From a11f6fd9cbbb62c627e43cb77c81b141518f2b41 Mon Sep 17 00:00:00 2001 From: Pavel Kunyavskiy Date: Mon, 28 Nov 2022 15:10:37 +0100 Subject: [PATCH] [K/N] Support of aligned fields For now only alignment by at most 8 is supported for instance fields. ^KT-54944 --- .../jetbrains/kotlin/backend/konan/Boxing.kt | 2 +- .../konan/descriptors/ClassLayoutBuilder.kt | 47 +++++++--- .../backend/konan/llvm/CodeGenerator.kt | 28 +++--- .../kotlin/backend/konan/llvm/IrToBitcode.kt | 30 +++---- .../backend/konan/llvm/KotlinStaticData.kt | 11 ++- .../backend/konan/llvm/LlvmDeclarations.kt | 88 ++++++++++++++----- .../kotlin/backend/konan/llvm/LlvmUtils.kt | 7 +- .../backend/konan/llvm/RTTIGenerator.kt | 18 ++-- .../kotlin/backend/konan/llvm/Runtime.kt | 3 + .../konan/serialization/KonanIrlinker.kt | 31 ++++--- .../runtime/src/legacymm/cpp/Memory.cpp | 1 + .../runtime/src/main/cpp/Alignment.hpp | 1 + .../runtime/src/main/cpp/ObjCExport.mm | 2 + .../src/main/cpp/ObjectTestSupport.hpp | 2 + kotlin-native/runtime/src/main/cpp/TypeInfo.h | 8 ++ 15 files changed, 193 insertions(+), 86 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt index 1724ae59339..241c19c1e7d 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Boxing.kt @@ -153,7 +153,7 @@ private fun initCache(cache: BoxCache, generationState: NativeGenerationState, c .setConstant(true) staticData.placeGlobal(rangeEndName, createConstant(llvmType, end), true) .setConstant(true) - val values = (start..end).map { staticData.createInitializer(kotlinType, createConstant(llvmType, it)) } + val values = (start..end).map { staticData.createConstKotlinObjectBody(kotlinType, createConstant(llvmType, it)) } staticData.placeGlobalArray(cacheName, llvmBoxType, values, true).also { it.setConstant(true) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt index d791732f133..59714964c38 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt @@ -5,6 +5,9 @@ package org.jetbrains.kotlin.backend.konan.descriptors +import llvm.LLVMABIAlignmentOfType +import llvm.LLVMABISizeOfType +import llvm.LLVMPreferredAlignmentOfType import llvm.LLVMStoreSizeOfType import org.jetbrains.kotlin.backend.common.lower.coroutines.getOrCreateFunctionWithContinuationStub import org.jetbrains.kotlin.backend.konan.* @@ -19,6 +22,7 @@ import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.expressions.IrClassReference import org.jetbrains.kotlin.ir.expressions.IrConst +import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid @@ -259,13 +263,33 @@ internal class GlobalHierarchyAnalysis(val context: Context, val irModule: IrMod } } -internal fun IrField.toFieldInfo(): ClassLayoutBuilder.FieldInfo { - val isConst = correspondingPropertySymbol?.owner?.isConst ?: false - require(!isConst || initializer?.expression is IrConst<*>) { "A const val field ${render()} must have constant initializer" } - return ClassLayoutBuilder.FieldInfo(name.asString(), type, isConst, this) +internal fun IrField.requiredAlignment(context: Context) : Int { + val llvm = context.generationState.llvm + val llvmType = type.toLLVMType(llvm) + val abiAlignment = if (llvmType == llvm.vector128Type) { + 8 // over-aligned objects are not supported now, and this worked somehow, so let's keep it as it for now + } else { + LLVMABIAlignmentOfType(llvm.runtime.targetData, llvmType) + } + return if (hasAnnotation(KonanFqNames.volatile)) { + val size = LLVMABISizeOfType(llvm.runtime.targetData, llvmType).toInt() + val alignment = maxOf(size, abiAlignment) + require(alignment % size == 0) { "Bad alignment of field ${render()}: abiAlignment = ${abiAlignment}, size = ${size}"} + require(alignment % abiAlignment == 0) { "Bad alignment of field ${render()}: abiAlignment = ${abiAlignment}, size = ${size}"} + alignment + } else { + abiAlignment + } } + internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { + private fun IrField.toFieldInfo(): FieldInfo { + val isConst = correspondingPropertySymbol?.owner?.isConst ?: false + require(!isConst || initializer?.expression is IrConst<*>) { "A const val field ${render()} must have constant initializer" } + return FieldInfo(name.asString(), type, isConst, symbol, requiredAlignment(context)) + } + val vtableEntries: List by lazy { require(!irClass.isInterface) @@ -400,8 +424,12 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { return context.getLayoutBuilder(superFunction.parentAsClass).itablePlace(superFunction) } - class FieldInfo(val name: String, val type: IrType, val isConst: Boolean, val irField: IrField?) { - var index = -1 + class FieldInfo(val name: String, val type: IrType, val isConst: Boolean, val irFieldSymbol: IrFieldSymbol, val alignment: Int) { + val irField: IrField? + get() = if (irFieldSymbol.isBound) irFieldSymbol.owner else null + init { + require(alignment.countOneBits() == 1) { "Alignment should be power of 2" } + } } /** @@ -413,7 +441,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { if (mappedField == fieldInfo.irField) fieldInfo else - mappedField!!.toFieldInfo().also { it.index = fieldInfo.index } + mappedField!!.toFieldInfo() } private var fields: List? = null @@ -432,9 +460,6 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { with(llvm) { LLVMStoreSizeOfType(runtime.targetData, it.type.toLLVMType(this)) } } - val superFieldsCount = 1 /* First field is ObjHeader */ + superFields.size - sortedDeclaredFields.forEachIndexed { index, field -> field.index = superFieldsCount + index } - return (superFields + sortedDeclaredFields).also { fields = it } } @@ -492,7 +517,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { require(context.config.cachedLibraries.isLibraryCached(moduleDeserializer.klib)) { "No IR and no cache for ${irClass.render()}" } - return moduleDeserializer.deserializeClassFields(irClass, outerThisField) + return moduleDeserializer.deserializeClassFields(irClass, outerThisField?.toFieldInfo()) } val declarations = irClass.declarations.toMutableList() diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index b6c4c67ab85..463999dadb8 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -261,9 +261,11 @@ internal class StackLocalsManagerImpl( if (context.memoryModel == MemoryModel.EXPERIMENTAL) alloca(kObjHeaderPtr) else null override fun alloc(irClass: IrClass, cleanFieldsExplicitly: Boolean): LLVMValueRef = with(functionGenerationContext) { - val type = llvmDeclarations.forClass(irClass).bodyType + val classInfo = llvmDeclarations.forClass(irClass) + val type = classInfo.bodyType val stackLocal = appendingTo(bbInitStackLocals) { val stackSlot = LLVMBuildAlloca(builder, type, "")!! + LLVMSetAlignment(stackSlot, classInfo.alignment) memset(bitcast(llvm.int8PtrType, stackSlot), 0, LLVMSizeOfTypeInBits(codegen.llvmTargetData, type).toInt() / 8) @@ -348,9 +350,9 @@ internal class StackLocalsManagerImpl( if (stackLocal.irClass.symbol == context.ir.symbols.array) call(llvm.zeroArrayRefsFunction, listOf(stackLocal.objHeaderPtr)) } else { - val type = llvmDeclarations.forClass(stackLocal.irClass).bodyType - for (field in context.getLayoutBuilder(stackLocal.irClass).getFields(llvm)) { - val fieldIndex = field.index + val info = llvmDeclarations.forClass(stackLocal.irClass) + val type = info.bodyType + for (fieldIndex in info.fieldIndices.values.sorted()) { val fieldType = LLVMStructGetTypeAtIndex(type, fieldIndex)!! if (isObjectType(fieldType)) { @@ -623,7 +625,7 @@ internal abstract class FunctionGenerationContext( private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean, isVolatile: Boolean = false, alignment: Int? = null) { - require(alignment == null || alignment == runtime.pointerAlignment) + require(alignment == null || alignment % runtime.pointerAlignment == 0) if (onStack) { require(!isVolatile) { "Stack ref update can't be volatile"} if (context.memoryModel == MemoryModel.STRICT) @@ -773,17 +775,17 @@ internal abstract class FunctionGenerationContext( } } - fun allocInstance(typeInfo: LLVMValueRef, lifetime: Lifetime, resultSlot: LLVMValueRef?): LLVMValueRef = + fun allocInstance(typeInfo: LLVMValueRef, lifetime: Lifetime, resultSlot: LLVMValueRef?) : LLVMValueRef = call(llvm.allocInstanceFunction, listOf(typeInfo), lifetime, resultSlot = resultSlot) fun allocInstance(irClass: IrClass, lifetime: Lifetime, stackLocalsManager: StackLocalsManager, resultSlot: LLVMValueRef?) = - if (lifetime == Lifetime.STACK) - stackLocalsManager.alloc(irClass, - // In case the allocation is not from the root scope, fields must be cleaned up explicitly, - // as the object might be being reused. - cleanFieldsExplicitly = stackLocalsManager != this.stackLocalsManager) - else - allocInstance(codegen.typeInfoForAllocation(irClass), lifetime, resultSlot) + if (lifetime == Lifetime.STACK) + stackLocalsManager.alloc(irClass, + // In case the allocation is not from the root scope, fields must be cleaned up explicitly, + // as the object might be being reused. + cleanFieldsExplicitly = stackLocalsManager != this.stackLocalsManager) + else + allocInstance(codegen.typeInfoForAllocation(irClass), lifetime, resultSlot) fun allocArray( irClass: IrClass, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 71615820def..d47e67a0031 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -775,7 +775,8 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, private fun getThreadLocalInitStateFor(container: IrDeclarationContainer): AddressAccess = llvm.initializersGenerationState.fileThreadLocalInitStates.getOrPut(container) { - codegen.addKotlinThreadLocal("state_thread_local$${container.initVariableSuffix}", llvm.int32Type).also { + codegen.addKotlinThreadLocal("state_thread_local$${container.initVariableSuffix}", llvm.int32Type, + LLVMPreferredAlignmentOfType(llvm.runtime.targetData, llvm.int32Type)).also { LLVMSetInitializer((it as GlobalAddressAccess).getAddress(null), llvm.int32(FILE_NOT_INITIALIZED)) } } @@ -1663,19 +1664,18 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, private fun evaluateGetField(value: IrGetField, resultSlot: LLVMValueRef?): LLVMValueRef { context.log { "evaluateGetField : ${ir2string(value)}" } - val alignment = when { - value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8 - else -> null - } + val alignment : Int val order = when { value.symbol.owner.hasAnnotation(KonanFqNames.volatile) -> LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent else -> null } + val fieldAddress: LLVMValueRef - val fieldAddress = when { + when { !value.symbol.owner.isStatic -> { - fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner) + fieldAddress = fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner) + alignment = context.generationState.llvmDeclarations.forField(value.symbol.owner).alignment } value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true -> { // TODO: probably can be removed, as they are inlined. @@ -1685,10 +1685,12 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, if (context.config.threadsAreAllowed && value.symbol.owner.isGlobalNonPrimitive(context)) { functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) } - generationState.llvmDeclarations + val info = generationState.llvmDeclarations .forStaticField(value.symbol.owner) + fieldAddress = info .storageAddressAccess .getAddress(functionGenerationContext) + alignment = info.alignment } } return functionGenerationContext.loadSlot( @@ -1739,6 +1741,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, val valueToAssign = evaluateExpression(value.value) val address: LLVMValueRef + val alignment: Int if (!value.symbol.owner.isStatic) { val thisPtr = evaluateExpression(value.receiver!!) assert(thisPtr.type == codegen.kObjHeaderPtr) { @@ -1754,19 +1757,16 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, functionGenerationContext.call(llvm.checkLifetimesConstraint, listOf(thisPtr, valueToAssign)) } address = fieldPtrOfClass(thisPtr, value.symbol.owner) + alignment = context.generationState.llvmDeclarations.forField(value.symbol.owner).alignment } else { assert(value.receiver == null) if (context.config.threadsAreAllowed && value.symbol.owner.storageKind(context) == FieldStorageKind.GLOBAL) functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) if (value.symbol.owner.shouldBeFrozen(context) && value.origin != ObjectClassLowering.IrStatementOriginFieldPreInit) functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler) - address = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress( - functionGenerationContext - ) - } - val alignment = when { - value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8 - else -> null + val info = generationState.llvmDeclarations.forStaticField(value.symbol.owner) + address = info.storageAddressAccess.getAddress(functionGenerationContext) + alignment = info.alignment } functionGenerationContext.storeAny( valueToAssign, address, false, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinStaticData.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinStaticData.kt index d7ab6b51102..18d1f0de06e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinStaticData.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinStaticData.kt @@ -71,10 +71,7 @@ internal class KotlinStaticData(override val generationState: NativeGenerationSt } fun createConstKotlinObject(type: IrClass, vararg fields: ConstValue): ConstPointer { - val typeInfo = type.typeInfoPtr - val objHeader = objHeader(typeInfo) - - val global = this.placeGlobal("", llvm.struct(objHeader, *fields)) + val global = this.placeGlobal("", createConstKotlinObjectBody(type, *fields)) global.setUnnamedAddr(true) global.setConstant(true) @@ -83,8 +80,10 @@ internal class KotlinStaticData(override val generationState: NativeGenerationSt return createRef(objHeaderPtr) } - fun createInitializer(type: IrClass, vararg fields: ConstValue): ConstValue = - llvm.struct(objHeader(type.typeInfoPtr), *fields) + fun createConstKotlinObjectBody(type: IrClass, vararg fields: ConstValue): ConstValue { + // TODO: handle padding here + return llvm.struct(objHeader(type.typeInfoPtr), *fields) + } fun createUniqueInstance( kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt index af565609107..6ab58ceaa3e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt @@ -11,9 +11,11 @@ import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleConstant import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic +import org.jetbrains.kotlin.backend.konan.descriptors.requiredAlignment import org.jetbrains.kotlin.backend.konan.ir.* import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid @@ -61,32 +63,68 @@ internal class ClassLlvmDeclarations( val typeInfoGlobal: StaticData.Global, val writableTypeInfoGlobal: StaticData.Global?, val typeInfo: ConstPointer, - val objCDeclarations: KotlinObjCClassLlvmDeclarations?) + val objCDeclarations: KotlinObjCClassLlvmDeclarations?, + val alignment: Int, + val fieldIndices: Map +) internal class KotlinObjCClassLlvmDeclarations( val classInfoGlobal: StaticData.Global, val bodyOffsetGlobal: StaticData.Global ) -internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMTypeRef) +internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMTypeRef, val alignment: Int) -internal class StaticFieldLlvmDeclarations(val storageAddressAccess: AddressAccess) +internal class StaticFieldLlvmDeclarations(val storageAddressAccess: AddressAccess, val alignment: Int) internal class UniqueLlvmDeclarations(val pointer: ConstPointer) -private fun ContextUtils.createClassBodyType(name: String, fields: List): LLVMTypeRef { - val fieldTypes = listOf(runtime.objHeaderType) + fields.map { it.type.toLLVMType(llvm) } - // TODO: consider adding synthetic ObjHeader field to Any. +internal data class ClassBodyAndAlignmentInfo( + val body: LLVMTypeRef, + val alignment: Int, + val fieldsIndices: Map +) +private fun ContextUtils.createClassBody(name: String, fields: List): ClassBodyAndAlignmentInfo { val classType = LLVMStructCreateNamed(LLVMGetModuleContext(llvm.module), name)!! + val packed = fields.any { LLVMABIAlignmentOfType(runtime.targetData, it.type.toLLVMType(llvm)) != it.alignment } + val alignment = maxOf(runtime.objectAlignment, fields.maxOfOrNull { it.alignment } ?: 0) + val indices = mutableMapOf() - // LLVMStructSetBody expects the struct to be properly aligned and will insert padding accordingly. In our case - // `allocInstance` returns 16x + 8 address, i.e. always misaligned for vector types. Workaround is to use packed struct. - val hasBigAlignment = fields.any { LLVMABIAlignmentOfType(runtime.targetData, it.type.toLLVMType(llvm)) > 8 } - val packed = if (hasBigAlignment) 1 else 0 - LLVMStructSetBody(classType, fieldTypes.toCValues(), fieldTypes.size, packed) + val fieldTypes = buildList { + var currentOffset = 0L + fun addAndCount(type: LLVMTypeRef) { + add(type) + currentOffset += LLVMStoreSizeOfType(runtime.targetData, type) + } + addAndCount(runtime.objHeaderType) + for (field in fields) { + if (packed) { + val offset = (currentOffset % field.alignment).toInt() + if (offset != 0) { + val toInsert = field.alignment - offset + addAndCount(LLVMArrayType(llvm.int8Type, toInsert)!!) + } + require(currentOffset % field.alignment == 0L) + } + indices[field.irFieldSymbol] = this.size + addAndCount(field.type.toLLVMType(llvm)) + } + } + LLVMStructSetBody(classType, fieldTypes.toCValues(), fieldTypes.size, if (packed) 1 else 0) - return classType + context.logMultiple { + +"$name has following fields:" + for (i in fieldTypes.indices) { + +" $i: ${llvmtype2string(fieldTypes[i])} at offset ${LLVMOffsetOfElement(runtime.targetData, classType, i)}" + } + +" Overall llvm alignment is ${LLVMABIAlignmentOfType(runtime.targetData, classType)}" + +" Overall required alignment is ${alignment}" + +" Overall size is ${LLVMABISizeOfType(runtime.targetData, classType)}" + +" Resulting type is ${llvmtype2string(classType)}" + } + + return ClassBodyAndAlignmentInfo(classType, alignment, indices) } private class DeclarationsGeneratorVisitor(override val generationState: NativeGenerationState) @@ -157,7 +195,12 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG val internalName = qualifyInternalName(declaration) val fields = context.getLayoutBuilder(declaration).getFields(llvm) - val bodyType = createClassBodyType("kclassbody:$internalName", fields) + val (bodyType, alignment, fieldIndices) = createClassBody("kclassbody:$internalName", fields) + + require(alignment == runtime.objectAlignment) { + "Over-aligned objects are not supported yet: expected alignment for ${declaration.fqNameWhenAvailable} is $alignment" + } + val typeInfoPtr: ConstPointer val typeInfoGlobal: StaticData.Global @@ -233,7 +276,7 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG it.setZeroInitializer() } - return ClassLlvmDeclarations(bodyType, typeInfoGlobal, writableTypeInfoGlobal, typeInfoPtr, objCDeclarations) + return ClassLlvmDeclarations(bodyType, typeInfoGlobal, writableTypeInfoGlobal, typeInfoPtr, objCDeclarations, alignment, fieldIndices) } private fun createUniqueDeclarations( @@ -273,6 +316,8 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG return KotlinObjCClassLlvmDeclarations(classInfoGlobal, bodyOffsetGlobal) } + private tailrec fun gcd(a: Long, b: Long) : Long = if (b == 0L) a else gcd(b, a % b) + override fun visitField(declaration: IrField) { super.visitField(declaration) @@ -281,29 +326,30 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG if (!containingClass.requiresRtti()) return val classDeclarations = (containingClass.metadata as? CodegenClassMetadata)?.llvm ?: error(containingClass.descriptor.toString()) - val allFields = context.getLayoutBuilder(containingClass).getFields(llvm) - val fieldInfo = allFields.firstOrNull { it.irField == declaration } ?: error("Field ${declaration.render()} is not found") + val index = classDeclarations.fieldIndices[declaration.symbol]!! declaration.metadata = CodegenInstanceFieldMetadata( declaration.metadata?.name, containingClass.konanLibrary, FieldLlvmDeclarations( - fieldInfo.index, - classDeclarations.bodyType + index, + classDeclarations.bodyType, + gcd(LLVMOffsetOfElement(llvm.runtime.targetData, classDeclarations.bodyType, index), llvm.runtime.objectAlignment.toLong()).toInt() ) ) } else { // Fields are module-private, so we use internal name: val name = "kvar:" + qualifyInternalName(declaration) + val alignmnet = declaration.requiredAlignment(context) val storage = if (declaration.storageKind(context) == FieldStorageKind.THREAD_LOCAL) { - addKotlinThreadLocal(name, declaration.type.toLLVMType(llvm)) + addKotlinThreadLocal(name, declaration.type.toLLVMType(llvm), alignmnet) } else { - addKotlinGlobal(name, declaration.type.toLLVMType(llvm), isExported = false) + addKotlinGlobal(name, declaration.type.toLLVMType(llvm), alignmnet, isExported = false) } declaration.metadata = CodegenStaticFieldMetadata( declaration.metadata?.name, declaration.konanLibrary, - StaticFieldLlvmDeclarations(storage) + StaticFieldLlvmDeclarations(storage, alignmnet) ) } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt index 3459484fd7d..4775b6cab37 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt @@ -241,23 +241,26 @@ internal class TLSAddressAccess(private val index: Int) : AddressAccess() { } } -internal fun ContextUtils.addKotlinThreadLocal(name: String, type: LLVMTypeRef): AddressAccess { +internal fun ContextUtils.addKotlinThreadLocal(name: String, type: LLVMTypeRef, alignment: Int): AddressAccess { return if (isObjectType(type)) { val index = llvm.tlsCount++ + require(llvm.runtime.pointerAlignment % alignment == 0) TLSAddressAccess(index) } else { // TODO: This will break if Workers get decoupled from host threads. GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also { LLVMSetThreadLocalMode(it, llvm.tlsMode) LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) + LLVMSetAlignment(it, alignment) }) } } -internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, isExported: Boolean): AddressAccess { +internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, alignment: Int, isExported: Boolean): AddressAccess { return GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also { if (!isExported) LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) + LLVMSetAlignment(it, alignment) }) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt index 8f151a4cf59..517749863d0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt @@ -99,9 +99,9 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState classId: Int, writableTypeInfo: ConstPointer?, associatedObjects: ConstPointer?, - processObjectInMark: ConstPointer?) : - - Struct( + processObjectInMark: ConstPointer?, + requiredAlignment: Int, + ) : Struct( runtime.typeInfoType, selfPtr, @@ -138,7 +138,8 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState associatedObjects, processObjectInMark, - ) + llvm.constInt32(requiredAlignment), + ) private fun kotlinStringLiteral(string: String?): ConstPointer = if (string == null) { NullPointer(runtime.objHeaderType) @@ -260,7 +261,8 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState processObjectInMark = when { irClass.symbol == context.ir.symbols.array -> constPointer(llvm.Kotlin_processArrayInMark.llvmValue) else -> genProcessObjectInMark(bodyType) - } + }, + requiredAlignment = llvmDeclarations.alignment ) val typeInfoGlobalValue = if (!irClass.typeInfoHasVtableAttached) { @@ -434,9 +436,10 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState class FieldRecord(val offset: Int, val type: Int, val name: String) val fields = context.getLayoutBuilder(irClass).getFields(llvm).map { + val index = llvmDeclarations.fieldIndices[it.irFieldSymbol]!! FieldRecord( - LLVMOffsetOfElement(llvmTargetData, bodyType, it.index).toInt(), - mapRuntimeType(LLVMStructGetTypeAtIndex(bodyType, it.index)!!), + LLVMOffsetOfElement(llvmTargetData, bodyType, index).toInt(), + mapRuntimeType(LLVMStructGetTypeAtIndex(bodyType, index)!!), it.name) } val offsetsPtr = staticData.placeGlobalConstArray("kextoff:$className", llvm.int32Type, @@ -563,6 +566,7 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState writableTypeInfo = writableTypeInfo, associatedObjects = null, processObjectInMark = genProcessObjectInMark(bodyType), + requiredAlignment = runtime.objectAlignment ), vtable) typeInfoWithVtableGlobal.setInitializer(typeInfoWithVtable) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt index e506ff0944b..9eb5e6720e0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt @@ -70,4 +70,7 @@ class Runtime(llvmContext: LLVMContextRef, bitcodeFile: String) { val pointerAlignment: Int by lazy { LLVMABIAlignmentOfType(targetData, objHeaderPtrType) } + + // Must match kObjectAlignment in runtime + val objectAlignment = 8 } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt index 78463bebaf4..8ad5ada2909 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt @@ -29,7 +29,7 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder import org.jetbrains.kotlin.backend.konan.descriptors.findPackage -import org.jetbrains.kotlin.backend.konan.descriptors.toFieldInfo +import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary import org.jetbrains.kotlin.backend.konan.ir.interop.IrProviderForCEnumAndCStructStubs import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.library.metadata.DeserializedKlibModuleOrigin @@ -44,6 +44,7 @@ import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyClass import org.jetbrains.kotlin.ir.expressions.IrBody import org.jetbrains.kotlin.ir.symbols.IrClassSymbol import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.symbols.impl.IrFieldSymbolImpl import org.jetbrains.kotlin.ir.symbols.impl.IrPublicSymbolBase import org.jetbrains.kotlin.ir.types.* import org.jetbrains.kotlin.ir.util.* @@ -147,7 +148,7 @@ internal object InlineFunctionBodyReferenceSerializer { // [binaryType] is needed in case a field is of a private inline class type (which can't be deserialized). // But it is safe to just set the field's type to the primitive type the inline class will be erased to. -class SerializedClassFieldInfo(val name: Int, val binaryType: Int, val type: Int, val flags: Int) { +class SerializedClassFieldInfo(val name: Int, val binaryType: Int, val type: Int, val flags: Int, val alignment: Int) { companion object { const val FLAG_IS_CONST = 1 } @@ -158,7 +159,7 @@ class SerializedClassFields(val file: Int, val classSignature: Int, val typePara internal object ClassFieldsSerializer { fun serialize(classFields: List): ByteArray { - val size = classFields.sumOf { Int.SIZE_BYTES * (5 + it.typeParameterSigs.size + it.fields.size * 4) } + val size = classFields.sumOf { Int.SIZE_BYTES * (5 + it.typeParameterSigs.size + it.fields.size * 5) } val stream = ByteArrayStream(ByteArray(size)) classFields.forEach { stream.writeInt(it.file) @@ -172,6 +173,7 @@ internal object ClassFieldsSerializer { stream.writeInt(field.binaryType) stream.writeInt(field.type) stream.writeInt(field.flags) + stream.writeInt(field.alignment) } } return stream.buf @@ -191,7 +193,8 @@ internal object ClassFieldsSerializer { val binaryType = stream.readInt() val type = stream.readInt() val flags = stream.readInt() - SerializedClassFieldInfo(name, binaryType, type, flags) + val alignment = stream.readInt() + SerializedClassFieldInfo(name, binaryType, type, flags, alignment) } result.add(SerializedClassFields(file, classSignature, typeParameterSigs, outerThisIndex, fields)) } @@ -630,7 +633,7 @@ internal class KonanIrLinker( val outerProtoClass = protoClasses[protoClasses.size - 2] val nameAndType = BinaryNameAndType.decode(outerProtoClass.thisReceiver.nameType) - SerializedClassFieldInfo(name = InvalidIndex, binaryType = InvalidIndex, nameAndType.typeIndex, flags = 0) + SerializedClassFieldInfo(name = InvalidIndex, binaryType = InvalidIndex, nameAndType.typeIndex, flags = 0, field.alignment) } else { val protoField = protoFieldsMap[field.name] ?: error("No proto for ${irField.render()}") val nameAndType = BinaryNameAndType.decode(protoField.nameType) @@ -647,7 +650,9 @@ internal class KonanIrLinker( if (with(KonanManglerIr) { (classifier as? IrClassSymbol)?.owner?.isExported(compatibleMode) } == false) InvalidIndex else nameAndType.typeIndex, - flags) + flags, + field.alignment + ) } }) } @@ -806,7 +811,7 @@ internal class KonanIrLinker( } } - fun deserializeClassFields(irClass: IrClass, outerThisField: IrField?): List { + fun deserializeClassFields(irClass: IrClass, outerThisFieldInfo: ClassLayoutBuilder.FieldInfo?): List { irClass.getPackageFragment() as? IrExternalPackageFragment ?: error("Expected an external package fragment for ${irClass.render()}") val signature = irClass.symbol.signature @@ -841,8 +846,10 @@ internal class KonanIrLinker( return serializedClassFields.fields.mapIndexed { index, field -> if (index == serializedClassFields.outerThisIndex) { require(irClass.isInner) { "Expected an inner class: ${irClass.render()}" } - require(outerThisField != null) { "For an inner class ${irClass.render()} there should be field" } - outerThisField.toFieldInfo() + require(outerThisFieldInfo != null) { "For an inner class ${irClass.render()} there should be field" } + outerThisFieldInfo.also { + require(it.alignment == field.alignment) { "Mismatched align information for outer this"} + } } else { val name = fileDeserializationState.fileReader.string(field.name) val type = when { @@ -862,7 +869,11 @@ internal class KonanIrLinker( } } ClassLayoutBuilder.FieldInfo( - name, type, isConst = (field.flags and SerializedClassFieldInfo.FLAG_IS_CONST) != 0, irField = null) + name, type, + isConst = (field.flags and SerializedClassFieldInfo.FLAG_IS_CONST) != 0, + irFieldSymbol = IrFieldSymbolImpl(), + alignment = field.alignment, + ) } } } diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 4a4a0e2f597..dfc3600e5cc 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -99,6 +99,7 @@ using container_size_t = size_t; // Granularity of arena container chunks. constexpr container_size_t kContainerAlignment = 1024; // Single object alignment. +// Must match objectAlignment in Runtime.kt constexpr container_size_t kObjectAlignment = 8; // Required e.g. for object size computations to be correct. diff --git a/kotlin-native/runtime/src/main/cpp/Alignment.hpp b/kotlin-native/runtime/src/main/cpp/Alignment.hpp index 4bfa10d5ccd..49e4febc007 100644 --- a/kotlin-native/runtime/src/main/cpp/Alignment.hpp +++ b/kotlin-native/runtime/src/main/cpp/Alignment.hpp @@ -11,6 +11,7 @@ namespace kotlin { +// Must match objectAlignment in Runtime.kt constexpr size_t kObjectAlignment = 8; template diff --git a/kotlin-native/runtime/src/main/cpp/ObjCExport.mm b/kotlin-native/runtime/src/main/cpp/ObjCExport.mm index 6cc7884aa80..b0c2d2a3ad1 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjCExport.mm +++ b/kotlin-native/runtime/src/main/cpp/ObjCExport.mm @@ -656,6 +656,7 @@ static const TypeInfo* createTypeInfo( result->superType_ = superType; if (fieldsInfo == nullptr) { result->instanceSize_ = superType->instanceSize_; + result->instanceAlignment_ = superType->instanceAlignment_; result->objOffsets_ = superType->objOffsets_; result->objOffsetsCount_ = superType->objOffsetsCount_; // So TF_IMMUTABLE can also be inherited: if ((superType->flags_ & TF_IMMUTABLE) != 0) { @@ -664,6 +665,7 @@ static const TypeInfo* createTypeInfo( result->processObjectInMark = superType->processObjectInMark; } else { result->instanceSize_ = fieldsInfo->instanceSize_; + result->instanceAlignment_ = fieldsInfo->instanceAlignment_; result->objOffsets_ = fieldsInfo->objOffsets_; result->objOffsetsCount_ = fieldsInfo->objOffsetsCount_; result->processObjectInMark = fieldsInfo->processObjectInMark; diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp index d9e7d0ab16b..eb3f5b39d6c 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp @@ -30,6 +30,7 @@ private: std_support::vector objOffsets_; int32_t objOffsetsCount_ = 0; int32_t flags_ = 0; + int32_t instanceAlignment_ = 8; const TypeInfo* superType_ = nullptr; void (*processObjectInMark_)(void*, ObjHeader*) = nullptr; }; @@ -90,6 +91,7 @@ public: typeInfo_.processObjectInMark = builder.processObjectInMark_; typeInfo_.flags_ = builder.flags_; typeInfo_.superType_ = builder.superType_; + typeInfo_.instanceAlignment_ = builder.instanceAlignment_; } TypeInfo* typeInfo() noexcept { return &typeInfo_; } diff --git a/kotlin-native/runtime/src/main/cpp/TypeInfo.h b/kotlin-native/runtime/src/main/cpp/TypeInfo.h index fc35e96eebd..12c040ecde6 100644 --- a/kotlin-native/runtime/src/main/cpp/TypeInfo.h +++ b/kotlin-native/runtime/src/main/cpp/TypeInfo.h @@ -90,6 +90,10 @@ struct InterfaceTableRecord { // This struct represents runtime type information and by itself is the compile time // constant. +// When adding a field here do not forget to adjust: +// 1. RTTIGenerator +// 2. ObjectTestSupport TypeInfoHolder +// 3. createTypeInfo in ObjcExport.mm struct TypeInfo { // Reference to self, to allow simple obtaining TypeInfo via meta-object. const TypeInfo* typeInfo_; @@ -136,6 +140,10 @@ struct TypeInfo { // TODO: Consider providing a generic traverse method instead. void (*processObjectInMark)(void* state, ObjHeader* object); + // Required alignment of instance + uint32_t instanceAlignment_; + + // vtable starts just after declared contents of the TypeInfo: // void* const vtable_[]; #ifdef __cplusplus