[K/N] Support of aligned fields

For now only alignment by at most 8 is supported for instance fields.

^KT-54944
This commit is contained in:
Pavel Kunyavskiy
2022-11-28 15:10:37 +01:00
committed by Space Team
parent 6da66649e7
commit a11f6fd9cb
15 changed files with 193 additions and 86 deletions
@@ -153,7 +153,7 @@ private fun initCache(cache: BoxCache, generationState: NativeGenerationState, c
.setConstant(true) .setConstant(true)
staticData.placeGlobal(rangeEndName, createConstant(llvmType, end), true) staticData.placeGlobal(rangeEndName, createConstant(llvmType, end), true)
.setConstant(true) .setConstant(true)
val values = (start..end).map { staticData.createInitializer(kotlinType, createConstant(llvmType, it)) } val values = (start..end).map { staticData.createConstKotlinObjectBody(kotlinType, createConstant(llvmType, it)) }
staticData.placeGlobalArray(cacheName, llvmBoxType, values, true).also { staticData.placeGlobalArray(cacheName, llvmBoxType, values, true).also {
it.setConstant(true) it.setConstant(true)
} }
@@ -5,6 +5,9 @@
package org.jetbrains.kotlin.backend.konan.descriptors package org.jetbrains.kotlin.backend.konan.descriptors
import llvm.LLVMABIAlignmentOfType
import llvm.LLVMABISizeOfType
import llvm.LLVMPreferredAlignmentOfType
import llvm.LLVMStoreSizeOfType import llvm.LLVMStoreSizeOfType
import org.jetbrains.kotlin.backend.common.lower.coroutines.getOrCreateFunctionWithContinuationStub import org.jetbrains.kotlin.backend.common.lower.coroutines.getOrCreateFunctionWithContinuationStub
import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.*
@@ -19,6 +22,7 @@ import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.expressions.IrClassReference import org.jetbrains.kotlin.ir.expressions.IrClassReference
import org.jetbrains.kotlin.ir.expressions.IrConst import org.jetbrains.kotlin.ir.expressions.IrConst
import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol
import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
@@ -259,13 +263,33 @@ internal class GlobalHierarchyAnalysis(val context: Context, val irModule: IrMod
} }
} }
internal fun IrField.toFieldInfo(): ClassLayoutBuilder.FieldInfo { internal fun IrField.requiredAlignment(context: Context) : Int {
val isConst = correspondingPropertySymbol?.owner?.isConst ?: false val llvm = context.generationState.llvm
require(!isConst || initializer?.expression is IrConst<*>) { "A const val field ${render()} must have constant initializer" } val llvmType = type.toLLVMType(llvm)
return ClassLayoutBuilder.FieldInfo(name.asString(), type, isConst, this) val abiAlignment = if (llvmType == llvm.vector128Type) {
8 // over-aligned objects are not supported now, and this worked somehow, so let's keep it as it for now
} else {
LLVMABIAlignmentOfType(llvm.runtime.targetData, llvmType)
}
return if (hasAnnotation(KonanFqNames.volatile)) {
val size = LLVMABISizeOfType(llvm.runtime.targetData, llvmType).toInt()
val alignment = maxOf(size, abiAlignment)
require(alignment % size == 0) { "Bad alignment of field ${render()}: abiAlignment = ${abiAlignment}, size = ${size}"}
require(alignment % abiAlignment == 0) { "Bad alignment of field ${render()}: abiAlignment = ${abiAlignment}, size = ${size}"}
alignment
} else {
abiAlignment
}
} }
internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) { internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
private fun IrField.toFieldInfo(): FieldInfo {
val isConst = correspondingPropertySymbol?.owner?.isConst ?: false
require(!isConst || initializer?.expression is IrConst<*>) { "A const val field ${render()} must have constant initializer" }
return FieldInfo(name.asString(), type, isConst, symbol, requiredAlignment(context))
}
val vtableEntries: List<OverriddenFunctionInfo> by lazy { val vtableEntries: List<OverriddenFunctionInfo> by lazy {
require(!irClass.isInterface) require(!irClass.isInterface)
@@ -400,8 +424,12 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
return context.getLayoutBuilder(superFunction.parentAsClass).itablePlace(superFunction) return context.getLayoutBuilder(superFunction.parentAsClass).itablePlace(superFunction)
} }
class FieldInfo(val name: String, val type: IrType, val isConst: Boolean, val irField: IrField?) { class FieldInfo(val name: String, val type: IrType, val isConst: Boolean, val irFieldSymbol: IrFieldSymbol, val alignment: Int) {
var index = -1 val irField: IrField?
get() = if (irFieldSymbol.isBound) irFieldSymbol.owner else null
init {
require(alignment.countOneBits() == 1) { "Alignment should be power of 2" }
}
} }
/** /**
@@ -413,7 +441,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
if (mappedField == fieldInfo.irField) if (mappedField == fieldInfo.irField)
fieldInfo fieldInfo
else else
mappedField!!.toFieldInfo().also { it.index = fieldInfo.index } mappedField!!.toFieldInfo()
} }
private var fields: List<FieldInfo>? = null private var fields: List<FieldInfo>? = null
@@ -432,9 +460,6 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
with(llvm) { LLVMStoreSizeOfType(runtime.targetData, it.type.toLLVMType(this)) } with(llvm) { LLVMStoreSizeOfType(runtime.targetData, it.type.toLLVMType(this)) }
} }
val superFieldsCount = 1 /* First field is ObjHeader */ + superFields.size
sortedDeclaredFields.forEachIndexed { index, field -> field.index = superFieldsCount + index }
return (superFields + sortedDeclaredFields).also { fields = it } return (superFields + sortedDeclaredFields).also { fields = it }
} }
@@ -492,7 +517,7 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
require(context.config.cachedLibraries.isLibraryCached(moduleDeserializer.klib)) { require(context.config.cachedLibraries.isLibraryCached(moduleDeserializer.klib)) {
"No IR and no cache for ${irClass.render()}" "No IR and no cache for ${irClass.render()}"
} }
return moduleDeserializer.deserializeClassFields(irClass, outerThisField) return moduleDeserializer.deserializeClassFields(irClass, outerThisField?.toFieldInfo())
} }
val declarations = irClass.declarations.toMutableList() val declarations = irClass.declarations.toMutableList()
@@ -261,9 +261,11 @@ internal class StackLocalsManagerImpl(
if (context.memoryModel == MemoryModel.EXPERIMENTAL) alloca(kObjHeaderPtr) else null if (context.memoryModel == MemoryModel.EXPERIMENTAL) alloca(kObjHeaderPtr) else null
override fun alloc(irClass: IrClass, cleanFieldsExplicitly: Boolean): LLVMValueRef = with(functionGenerationContext) { override fun alloc(irClass: IrClass, cleanFieldsExplicitly: Boolean): LLVMValueRef = with(functionGenerationContext) {
val type = llvmDeclarations.forClass(irClass).bodyType val classInfo = llvmDeclarations.forClass(irClass)
val type = classInfo.bodyType
val stackLocal = appendingTo(bbInitStackLocals) { val stackLocal = appendingTo(bbInitStackLocals) {
val stackSlot = LLVMBuildAlloca(builder, type, "")!! val stackSlot = LLVMBuildAlloca(builder, type, "")!!
LLVMSetAlignment(stackSlot, classInfo.alignment)
memset(bitcast(llvm.int8PtrType, stackSlot), 0, LLVMSizeOfTypeInBits(codegen.llvmTargetData, type).toInt() / 8) memset(bitcast(llvm.int8PtrType, stackSlot), 0, LLVMSizeOfTypeInBits(codegen.llvmTargetData, type).toInt() / 8)
@@ -348,9 +350,9 @@ internal class StackLocalsManagerImpl(
if (stackLocal.irClass.symbol == context.ir.symbols.array) if (stackLocal.irClass.symbol == context.ir.symbols.array)
call(llvm.zeroArrayRefsFunction, listOf(stackLocal.objHeaderPtr)) call(llvm.zeroArrayRefsFunction, listOf(stackLocal.objHeaderPtr))
} else { } else {
val type = llvmDeclarations.forClass(stackLocal.irClass).bodyType val info = llvmDeclarations.forClass(stackLocal.irClass)
for (field in context.getLayoutBuilder(stackLocal.irClass).getFields(llvm)) { val type = info.bodyType
val fieldIndex = field.index for (fieldIndex in info.fieldIndices.values.sorted()) {
val fieldType = LLVMStructGetTypeAtIndex(type, fieldIndex)!! val fieldType = LLVMStructGetTypeAtIndex(type, fieldIndex)!!
if (isObjectType(fieldType)) { if (isObjectType(fieldType)) {
@@ -623,7 +625,7 @@ internal abstract class FunctionGenerationContext(
private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean, private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean,
isVolatile: Boolean = false, alignment: Int? = null) { isVolatile: Boolean = false, alignment: Int? = null) {
require(alignment == null || alignment == runtime.pointerAlignment) require(alignment == null || alignment % runtime.pointerAlignment == 0)
if (onStack) { if (onStack) {
require(!isVolatile) { "Stack ref update can't be volatile"} require(!isVolatile) { "Stack ref update can't be volatile"}
if (context.memoryModel == MemoryModel.STRICT) if (context.memoryModel == MemoryModel.STRICT)
@@ -773,17 +775,17 @@ internal abstract class FunctionGenerationContext(
} }
} }
fun allocInstance(typeInfo: LLVMValueRef, lifetime: Lifetime, resultSlot: LLVMValueRef?): LLVMValueRef = fun allocInstance(typeInfo: LLVMValueRef, lifetime: Lifetime, resultSlot: LLVMValueRef?) : LLVMValueRef =
call(llvm.allocInstanceFunction, listOf(typeInfo), lifetime, resultSlot = resultSlot) call(llvm.allocInstanceFunction, listOf(typeInfo), lifetime, resultSlot = resultSlot)
fun allocInstance(irClass: IrClass, lifetime: Lifetime, stackLocalsManager: StackLocalsManager, resultSlot: LLVMValueRef?) = fun allocInstance(irClass: IrClass, lifetime: Lifetime, stackLocalsManager: StackLocalsManager, resultSlot: LLVMValueRef?) =
if (lifetime == Lifetime.STACK) if (lifetime == Lifetime.STACK)
stackLocalsManager.alloc(irClass, stackLocalsManager.alloc(irClass,
// In case the allocation is not from the root scope, fields must be cleaned up explicitly, // In case the allocation is not from the root scope, fields must be cleaned up explicitly,
// as the object might be being reused. // as the object might be being reused.
cleanFieldsExplicitly = stackLocalsManager != this.stackLocalsManager) cleanFieldsExplicitly = stackLocalsManager != this.stackLocalsManager)
else else
allocInstance(codegen.typeInfoForAllocation(irClass), lifetime, resultSlot) allocInstance(codegen.typeInfoForAllocation(irClass), lifetime, resultSlot)
fun allocArray( fun allocArray(
irClass: IrClass, irClass: IrClass,
@@ -775,7 +775,8 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
private fun getThreadLocalInitStateFor(container: IrDeclarationContainer): AddressAccess = private fun getThreadLocalInitStateFor(container: IrDeclarationContainer): AddressAccess =
llvm.initializersGenerationState.fileThreadLocalInitStates.getOrPut(container) { llvm.initializersGenerationState.fileThreadLocalInitStates.getOrPut(container) {
codegen.addKotlinThreadLocal("state_thread_local$${container.initVariableSuffix}", llvm.int32Type).also { codegen.addKotlinThreadLocal("state_thread_local$${container.initVariableSuffix}", llvm.int32Type,
LLVMPreferredAlignmentOfType(llvm.runtime.targetData, llvm.int32Type)).also {
LLVMSetInitializer((it as GlobalAddressAccess).getAddress(null), llvm.int32(FILE_NOT_INITIALIZED)) LLVMSetInitializer((it as GlobalAddressAccess).getAddress(null), llvm.int32(FILE_NOT_INITIALIZED))
} }
} }
@@ -1663,19 +1664,18 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
private fun evaluateGetField(value: IrGetField, resultSlot: LLVMValueRef?): LLVMValueRef { private fun evaluateGetField(value: IrGetField, resultSlot: LLVMValueRef?): LLVMValueRef {
context.log { "evaluateGetField : ${ir2string(value)}" } context.log { "evaluateGetField : ${ir2string(value)}" }
val alignment = when { val alignment : Int
value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8
else -> null
}
val order = when { val order = when {
value.symbol.owner.hasAnnotation(KonanFqNames.volatile) -> value.symbol.owner.hasAnnotation(KonanFqNames.volatile) ->
LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent
else -> null else -> null
} }
val fieldAddress: LLVMValueRef
val fieldAddress = when { when {
!value.symbol.owner.isStatic -> { !value.symbol.owner.isStatic -> {
fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner) fieldAddress = fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner)
alignment = context.generationState.llvmDeclarations.forField(value.symbol.owner).alignment
} }
value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true -> { value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true -> {
// TODO: probably can be removed, as they are inlined. // TODO: probably can be removed, as they are inlined.
@@ -1685,10 +1685,12 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
if (context.config.threadsAreAllowed && value.symbol.owner.isGlobalNonPrimitive(context)) { if (context.config.threadsAreAllowed && value.symbol.owner.isGlobalNonPrimitive(context)) {
functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler)
} }
generationState.llvmDeclarations val info = generationState.llvmDeclarations
.forStaticField(value.symbol.owner) .forStaticField(value.symbol.owner)
fieldAddress = info
.storageAddressAccess .storageAddressAccess
.getAddress(functionGenerationContext) .getAddress(functionGenerationContext)
alignment = info.alignment
} }
} }
return functionGenerationContext.loadSlot( return functionGenerationContext.loadSlot(
@@ -1739,6 +1741,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
val valueToAssign = evaluateExpression(value.value) val valueToAssign = evaluateExpression(value.value)
val address: LLVMValueRef val address: LLVMValueRef
val alignment: Int
if (!value.symbol.owner.isStatic) { if (!value.symbol.owner.isStatic) {
val thisPtr = evaluateExpression(value.receiver!!) val thisPtr = evaluateExpression(value.receiver!!)
assert(thisPtr.type == codegen.kObjHeaderPtr) { assert(thisPtr.type == codegen.kObjHeaderPtr) {
@@ -1754,19 +1757,16 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
functionGenerationContext.call(llvm.checkLifetimesConstraint, listOf(thisPtr, valueToAssign)) functionGenerationContext.call(llvm.checkLifetimesConstraint, listOf(thisPtr, valueToAssign))
} }
address = fieldPtrOfClass(thisPtr, value.symbol.owner) address = fieldPtrOfClass(thisPtr, value.symbol.owner)
alignment = context.generationState.llvmDeclarations.forField(value.symbol.owner).alignment
} else { } else {
assert(value.receiver == null) assert(value.receiver == null)
if (context.config.threadsAreAllowed && value.symbol.owner.storageKind(context) == FieldStorageKind.GLOBAL) if (context.config.threadsAreAllowed && value.symbol.owner.storageKind(context) == FieldStorageKind.GLOBAL)
functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler)
if (value.symbol.owner.shouldBeFrozen(context) && value.origin != ObjectClassLowering.IrStatementOriginFieldPreInit) if (value.symbol.owner.shouldBeFrozen(context) && value.origin != ObjectClassLowering.IrStatementOriginFieldPreInit)
functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler) functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler)
address = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress( val info = generationState.llvmDeclarations.forStaticField(value.symbol.owner)
functionGenerationContext address = info.storageAddressAccess.getAddress(functionGenerationContext)
) alignment = info.alignment
}
val alignment = when {
value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8
else -> null
} }
functionGenerationContext.storeAny( functionGenerationContext.storeAny(
valueToAssign, address, false, valueToAssign, address, false,
@@ -71,10 +71,7 @@ internal class KotlinStaticData(override val generationState: NativeGenerationSt
} }
fun createConstKotlinObject(type: IrClass, vararg fields: ConstValue): ConstPointer { fun createConstKotlinObject(type: IrClass, vararg fields: ConstValue): ConstPointer {
val typeInfo = type.typeInfoPtr val global = this.placeGlobal("", createConstKotlinObjectBody(type, *fields))
val objHeader = objHeader(typeInfo)
val global = this.placeGlobal("", llvm.struct(objHeader, *fields))
global.setUnnamedAddr(true) global.setUnnamedAddr(true)
global.setConstant(true) global.setConstant(true)
@@ -83,8 +80,10 @@ internal class KotlinStaticData(override val generationState: NativeGenerationSt
return createRef(objHeaderPtr) return createRef(objHeaderPtr)
} }
fun createInitializer(type: IrClass, vararg fields: ConstValue): ConstValue = fun createConstKotlinObjectBody(type: IrClass, vararg fields: ConstValue): ConstValue {
llvm.struct(objHeader(type.typeInfoPtr), *fields) // TODO: handle padding here
return llvm.struct(objHeader(type.typeInfoPtr), *fields)
}
fun createUniqueInstance( fun createUniqueInstance(
kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer { kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer {
@@ -11,9 +11,11 @@ import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleConstant
import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder
import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic
import org.jetbrains.kotlin.backend.konan.descriptors.requiredAlignment
import org.jetbrains.kotlin.backend.konan.ir.* import org.jetbrains.kotlin.backend.konan.ir.*
import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol
import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
@@ -61,32 +63,68 @@ internal class ClassLlvmDeclarations(
val typeInfoGlobal: StaticData.Global, val typeInfoGlobal: StaticData.Global,
val writableTypeInfoGlobal: StaticData.Global?, val writableTypeInfoGlobal: StaticData.Global?,
val typeInfo: ConstPointer, val typeInfo: ConstPointer,
val objCDeclarations: KotlinObjCClassLlvmDeclarations?) val objCDeclarations: KotlinObjCClassLlvmDeclarations?,
val alignment: Int,
val fieldIndices: Map<IrFieldSymbol, Int>
)
internal class KotlinObjCClassLlvmDeclarations( internal class KotlinObjCClassLlvmDeclarations(
val classInfoGlobal: StaticData.Global, val classInfoGlobal: StaticData.Global,
val bodyOffsetGlobal: StaticData.Global val bodyOffsetGlobal: StaticData.Global
) )
internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMTypeRef) internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMTypeRef, val alignment: Int)
internal class StaticFieldLlvmDeclarations(val storageAddressAccess: AddressAccess) internal class StaticFieldLlvmDeclarations(val storageAddressAccess: AddressAccess, val alignment: Int)
internal class UniqueLlvmDeclarations(val pointer: ConstPointer) internal class UniqueLlvmDeclarations(val pointer: ConstPointer)
private fun ContextUtils.createClassBodyType(name: String, fields: List<ClassLayoutBuilder.FieldInfo>): LLVMTypeRef { internal data class ClassBodyAndAlignmentInfo(
val fieldTypes = listOf(runtime.objHeaderType) + fields.map { it.type.toLLVMType(llvm) } val body: LLVMTypeRef,
// TODO: consider adding synthetic ObjHeader field to Any. val alignment: Int,
val fieldsIndices: Map<IrFieldSymbol, Int>
)
private fun ContextUtils.createClassBody(name: String, fields: List<ClassLayoutBuilder.FieldInfo>): ClassBodyAndAlignmentInfo {
val classType = LLVMStructCreateNamed(LLVMGetModuleContext(llvm.module), name)!! val classType = LLVMStructCreateNamed(LLVMGetModuleContext(llvm.module), name)!!
val packed = fields.any { LLVMABIAlignmentOfType(runtime.targetData, it.type.toLLVMType(llvm)) != it.alignment }
val alignment = maxOf(runtime.objectAlignment, fields.maxOfOrNull { it.alignment } ?: 0)
val indices = mutableMapOf<IrFieldSymbol, Int>()
// LLVMStructSetBody expects the struct to be properly aligned and will insert padding accordingly. In our case val fieldTypes = buildList {
// `allocInstance` returns 16x + 8 address, i.e. always misaligned for vector types. Workaround is to use packed struct. var currentOffset = 0L
val hasBigAlignment = fields.any { LLVMABIAlignmentOfType(runtime.targetData, it.type.toLLVMType(llvm)) > 8 } fun addAndCount(type: LLVMTypeRef) {
val packed = if (hasBigAlignment) 1 else 0 add(type)
LLVMStructSetBody(classType, fieldTypes.toCValues(), fieldTypes.size, packed) currentOffset += LLVMStoreSizeOfType(runtime.targetData, type)
}
addAndCount(runtime.objHeaderType)
for (field in fields) {
if (packed) {
val offset = (currentOffset % field.alignment).toInt()
if (offset != 0) {
val toInsert = field.alignment - offset
addAndCount(LLVMArrayType(llvm.int8Type, toInsert)!!)
}
require(currentOffset % field.alignment == 0L)
}
indices[field.irFieldSymbol] = this.size
addAndCount(field.type.toLLVMType(llvm))
}
}
LLVMStructSetBody(classType, fieldTypes.toCValues(), fieldTypes.size, if (packed) 1 else 0)
return classType context.logMultiple {
+"$name has following fields:"
for (i in fieldTypes.indices) {
+" $i: ${llvmtype2string(fieldTypes[i])} at offset ${LLVMOffsetOfElement(runtime.targetData, classType, i)}"
}
+" Overall llvm alignment is ${LLVMABIAlignmentOfType(runtime.targetData, classType)}"
+" Overall required alignment is ${alignment}"
+" Overall size is ${LLVMABISizeOfType(runtime.targetData, classType)}"
+" Resulting type is ${llvmtype2string(classType)}"
}
return ClassBodyAndAlignmentInfo(classType, alignment, indices)
} }
private class DeclarationsGeneratorVisitor(override val generationState: NativeGenerationState) private class DeclarationsGeneratorVisitor(override val generationState: NativeGenerationState)
@@ -157,7 +195,12 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG
val internalName = qualifyInternalName(declaration) val internalName = qualifyInternalName(declaration)
val fields = context.getLayoutBuilder(declaration).getFields(llvm) val fields = context.getLayoutBuilder(declaration).getFields(llvm)
val bodyType = createClassBodyType("kclassbody:$internalName", fields) val (bodyType, alignment, fieldIndices) = createClassBody("kclassbody:$internalName", fields)
require(alignment == runtime.objectAlignment) {
"Over-aligned objects are not supported yet: expected alignment for ${declaration.fqNameWhenAvailable} is $alignment"
}
val typeInfoPtr: ConstPointer val typeInfoPtr: ConstPointer
val typeInfoGlobal: StaticData.Global val typeInfoGlobal: StaticData.Global
@@ -233,7 +276,7 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG
it.setZeroInitializer() it.setZeroInitializer()
} }
return ClassLlvmDeclarations(bodyType, typeInfoGlobal, writableTypeInfoGlobal, typeInfoPtr, objCDeclarations) return ClassLlvmDeclarations(bodyType, typeInfoGlobal, writableTypeInfoGlobal, typeInfoPtr, objCDeclarations, alignment, fieldIndices)
} }
private fun createUniqueDeclarations( private fun createUniqueDeclarations(
@@ -273,6 +316,8 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG
return KotlinObjCClassLlvmDeclarations(classInfoGlobal, bodyOffsetGlobal) return KotlinObjCClassLlvmDeclarations(classInfoGlobal, bodyOffsetGlobal)
} }
private tailrec fun gcd(a: Long, b: Long) : Long = if (b == 0L) a else gcd(b, a % b)
override fun visitField(declaration: IrField) { override fun visitField(declaration: IrField) {
super.visitField(declaration) super.visitField(declaration)
@@ -281,29 +326,30 @@ private class DeclarationsGeneratorVisitor(override val generationState: NativeG
if (!containingClass.requiresRtti()) return if (!containingClass.requiresRtti()) return
val classDeclarations = (containingClass.metadata as? CodegenClassMetadata)?.llvm val classDeclarations = (containingClass.metadata as? CodegenClassMetadata)?.llvm
?: error(containingClass.descriptor.toString()) ?: error(containingClass.descriptor.toString())
val allFields = context.getLayoutBuilder(containingClass).getFields(llvm) val index = classDeclarations.fieldIndices[declaration.symbol]!!
val fieldInfo = allFields.firstOrNull { it.irField == declaration } ?: error("Field ${declaration.render()} is not found")
declaration.metadata = CodegenInstanceFieldMetadata( declaration.metadata = CodegenInstanceFieldMetadata(
declaration.metadata?.name, declaration.metadata?.name,
containingClass.konanLibrary, containingClass.konanLibrary,
FieldLlvmDeclarations( FieldLlvmDeclarations(
fieldInfo.index, index,
classDeclarations.bodyType classDeclarations.bodyType,
gcd(LLVMOffsetOfElement(llvm.runtime.targetData, classDeclarations.bodyType, index), llvm.runtime.objectAlignment.toLong()).toInt()
) )
) )
} else { } else {
// Fields are module-private, so we use internal name: // Fields are module-private, so we use internal name:
val name = "kvar:" + qualifyInternalName(declaration) val name = "kvar:" + qualifyInternalName(declaration)
val alignmnet = declaration.requiredAlignment(context)
val storage = if (declaration.storageKind(context) == FieldStorageKind.THREAD_LOCAL) { val storage = if (declaration.storageKind(context) == FieldStorageKind.THREAD_LOCAL) {
addKotlinThreadLocal(name, declaration.type.toLLVMType(llvm)) addKotlinThreadLocal(name, declaration.type.toLLVMType(llvm), alignmnet)
} else { } else {
addKotlinGlobal(name, declaration.type.toLLVMType(llvm), isExported = false) addKotlinGlobal(name, declaration.type.toLLVMType(llvm), alignmnet, isExported = false)
} }
declaration.metadata = CodegenStaticFieldMetadata( declaration.metadata = CodegenStaticFieldMetadata(
declaration.metadata?.name, declaration.metadata?.name,
declaration.konanLibrary, declaration.konanLibrary,
StaticFieldLlvmDeclarations(storage) StaticFieldLlvmDeclarations(storage, alignmnet)
) )
} }
} }
@@ -241,23 +241,26 @@ internal class TLSAddressAccess(private val index: Int) : AddressAccess() {
} }
} }
internal fun ContextUtils.addKotlinThreadLocal(name: String, type: LLVMTypeRef): AddressAccess { internal fun ContextUtils.addKotlinThreadLocal(name: String, type: LLVMTypeRef, alignment: Int): AddressAccess {
return if (isObjectType(type)) { return if (isObjectType(type)) {
val index = llvm.tlsCount++ val index = llvm.tlsCount++
require(llvm.runtime.pointerAlignment % alignment == 0)
TLSAddressAccess(index) TLSAddressAccess(index)
} else { } else {
// TODO: This will break if Workers get decoupled from host threads. // TODO: This will break if Workers get decoupled from host threads.
GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also { GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also {
LLVMSetThreadLocalMode(it, llvm.tlsMode) LLVMSetThreadLocalMode(it, llvm.tlsMode)
LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage)
LLVMSetAlignment(it, alignment)
}) })
} }
} }
internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, isExported: Boolean): AddressAccess { internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, alignment: Int, isExported: Boolean): AddressAccess {
return GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also { return GlobalAddressAccess(LLVMAddGlobal(llvm.module, type, name)!!.also {
if (!isExported) if (!isExported)
LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage)
LLVMSetAlignment(it, alignment)
}) })
} }
@@ -99,9 +99,9 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState
classId: Int, classId: Int,
writableTypeInfo: ConstPointer?, writableTypeInfo: ConstPointer?,
associatedObjects: ConstPointer?, associatedObjects: ConstPointer?,
processObjectInMark: ConstPointer?) : processObjectInMark: ConstPointer?,
requiredAlignment: Int,
Struct( ) : Struct(
runtime.typeInfoType, runtime.typeInfoType,
selfPtr, selfPtr,
@@ -138,7 +138,8 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState
associatedObjects, associatedObjects,
processObjectInMark, processObjectInMark,
) llvm.constInt32(requiredAlignment),
)
private fun kotlinStringLiteral(string: String?): ConstPointer = if (string == null) { private fun kotlinStringLiteral(string: String?): ConstPointer = if (string == null) {
NullPointer(runtime.objHeaderType) NullPointer(runtime.objHeaderType)
@@ -260,7 +261,8 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState
processObjectInMark = when { processObjectInMark = when {
irClass.symbol == context.ir.symbols.array -> constPointer(llvm.Kotlin_processArrayInMark.llvmValue) irClass.symbol == context.ir.symbols.array -> constPointer(llvm.Kotlin_processArrayInMark.llvmValue)
else -> genProcessObjectInMark(bodyType) else -> genProcessObjectInMark(bodyType)
} },
requiredAlignment = llvmDeclarations.alignment
) )
val typeInfoGlobalValue = if (!irClass.typeInfoHasVtableAttached) { val typeInfoGlobalValue = if (!irClass.typeInfoHasVtableAttached) {
@@ -434,9 +436,10 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState
class FieldRecord(val offset: Int, val type: Int, val name: String) class FieldRecord(val offset: Int, val type: Int, val name: String)
val fields = context.getLayoutBuilder(irClass).getFields(llvm).map { val fields = context.getLayoutBuilder(irClass).getFields(llvm).map {
val index = llvmDeclarations.fieldIndices[it.irFieldSymbol]!!
FieldRecord( FieldRecord(
LLVMOffsetOfElement(llvmTargetData, bodyType, it.index).toInt(), LLVMOffsetOfElement(llvmTargetData, bodyType, index).toInt(),
mapRuntimeType(LLVMStructGetTypeAtIndex(bodyType, it.index)!!), mapRuntimeType(LLVMStructGetTypeAtIndex(bodyType, index)!!),
it.name) it.name)
} }
val offsetsPtr = staticData.placeGlobalConstArray("kextoff:$className", llvm.int32Type, val offsetsPtr = staticData.placeGlobalConstArray("kextoff:$className", llvm.int32Type,
@@ -563,6 +566,7 @@ internal class RTTIGenerator(override val generationState: NativeGenerationState
writableTypeInfo = writableTypeInfo, writableTypeInfo = writableTypeInfo,
associatedObjects = null, associatedObjects = null,
processObjectInMark = genProcessObjectInMark(bodyType), processObjectInMark = genProcessObjectInMark(bodyType),
requiredAlignment = runtime.objectAlignment
), vtable) ), vtable)
typeInfoWithVtableGlobal.setInitializer(typeInfoWithVtable) typeInfoWithVtableGlobal.setInitializer(typeInfoWithVtable)
@@ -70,4 +70,7 @@ class Runtime(llvmContext: LLVMContextRef, bitcodeFile: String) {
val pointerAlignment: Int by lazy { val pointerAlignment: Int by lazy {
LLVMABIAlignmentOfType(targetData, objHeaderPtrType) LLVMABIAlignmentOfType(targetData, objHeaderPtrType)
} }
// Must match kObjectAlignment in runtime
val objectAlignment = 8
} }
@@ -29,7 +29,7 @@ import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.descriptors.*
import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder import org.jetbrains.kotlin.backend.konan.descriptors.ClassLayoutBuilder
import org.jetbrains.kotlin.backend.konan.descriptors.findPackage import org.jetbrains.kotlin.backend.konan.descriptors.findPackage
import org.jetbrains.kotlin.backend.konan.descriptors.toFieldInfo import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary
import org.jetbrains.kotlin.backend.konan.ir.interop.IrProviderForCEnumAndCStructStubs import org.jetbrains.kotlin.backend.konan.ir.interop.IrProviderForCEnumAndCStructStubs
import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.library.metadata.DeserializedKlibModuleOrigin import org.jetbrains.kotlin.library.metadata.DeserializedKlibModuleOrigin
@@ -44,6 +44,7 @@ import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyClass
import org.jetbrains.kotlin.ir.expressions.IrBody import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.symbols.IrClassSymbol import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.symbols.impl.IrFieldSymbolImpl
import org.jetbrains.kotlin.ir.symbols.impl.IrPublicSymbolBase import org.jetbrains.kotlin.ir.symbols.impl.IrPublicSymbolBase
import org.jetbrains.kotlin.ir.types.* import org.jetbrains.kotlin.ir.types.*
import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.util.*
@@ -147,7 +148,7 @@ internal object InlineFunctionBodyReferenceSerializer {
// [binaryType] is needed in case a field is of a private inline class type (which can't be deserialized). // [binaryType] is needed in case a field is of a private inline class type (which can't be deserialized).
// But it is safe to just set the field's type to the primitive type the inline class will be erased to. // But it is safe to just set the field's type to the primitive type the inline class will be erased to.
class SerializedClassFieldInfo(val name: Int, val binaryType: Int, val type: Int, val flags: Int) { class SerializedClassFieldInfo(val name: Int, val binaryType: Int, val type: Int, val flags: Int, val alignment: Int) {
companion object { companion object {
const val FLAG_IS_CONST = 1 const val FLAG_IS_CONST = 1
} }
@@ -158,7 +159,7 @@ class SerializedClassFields(val file: Int, val classSignature: Int, val typePara
internal object ClassFieldsSerializer { internal object ClassFieldsSerializer {
fun serialize(classFields: List<SerializedClassFields>): ByteArray { fun serialize(classFields: List<SerializedClassFields>): ByteArray {
val size = classFields.sumOf { Int.SIZE_BYTES * (5 + it.typeParameterSigs.size + it.fields.size * 4) } val size = classFields.sumOf { Int.SIZE_BYTES * (5 + it.typeParameterSigs.size + it.fields.size * 5) }
val stream = ByteArrayStream(ByteArray(size)) val stream = ByteArrayStream(ByteArray(size))
classFields.forEach { classFields.forEach {
stream.writeInt(it.file) stream.writeInt(it.file)
@@ -172,6 +173,7 @@ internal object ClassFieldsSerializer {
stream.writeInt(field.binaryType) stream.writeInt(field.binaryType)
stream.writeInt(field.type) stream.writeInt(field.type)
stream.writeInt(field.flags) stream.writeInt(field.flags)
stream.writeInt(field.alignment)
} }
} }
return stream.buf return stream.buf
@@ -191,7 +193,8 @@ internal object ClassFieldsSerializer {
val binaryType = stream.readInt() val binaryType = stream.readInt()
val type = stream.readInt() val type = stream.readInt()
val flags = stream.readInt() val flags = stream.readInt()
SerializedClassFieldInfo(name, binaryType, type, flags) val alignment = stream.readInt()
SerializedClassFieldInfo(name, binaryType, type, flags, alignment)
} }
result.add(SerializedClassFields(file, classSignature, typeParameterSigs, outerThisIndex, fields)) result.add(SerializedClassFields(file, classSignature, typeParameterSigs, outerThisIndex, fields))
} }
@@ -630,7 +633,7 @@ internal class KonanIrLinker(
val outerProtoClass = protoClasses[protoClasses.size - 2] val outerProtoClass = protoClasses[protoClasses.size - 2]
val nameAndType = BinaryNameAndType.decode(outerProtoClass.thisReceiver.nameType) val nameAndType = BinaryNameAndType.decode(outerProtoClass.thisReceiver.nameType)
SerializedClassFieldInfo(name = InvalidIndex, binaryType = InvalidIndex, nameAndType.typeIndex, flags = 0) SerializedClassFieldInfo(name = InvalidIndex, binaryType = InvalidIndex, nameAndType.typeIndex, flags = 0, field.alignment)
} else { } else {
val protoField = protoFieldsMap[field.name] ?: error("No proto for ${irField.render()}") val protoField = protoFieldsMap[field.name] ?: error("No proto for ${irField.render()}")
val nameAndType = BinaryNameAndType.decode(protoField.nameType) val nameAndType = BinaryNameAndType.decode(protoField.nameType)
@@ -647,7 +650,9 @@ internal class KonanIrLinker(
if (with(KonanManglerIr) { (classifier as? IrClassSymbol)?.owner?.isExported(compatibleMode) } == false) if (with(KonanManglerIr) { (classifier as? IrClassSymbol)?.owner?.isExported(compatibleMode) } == false)
InvalidIndex InvalidIndex
else nameAndType.typeIndex, else nameAndType.typeIndex,
flags) flags,
field.alignment
)
} }
}) })
} }
@@ -806,7 +811,7 @@ internal class KonanIrLinker(
} }
} }
fun deserializeClassFields(irClass: IrClass, outerThisField: IrField?): List<ClassLayoutBuilder.FieldInfo> { fun deserializeClassFields(irClass: IrClass, outerThisFieldInfo: ClassLayoutBuilder.FieldInfo?): List<ClassLayoutBuilder.FieldInfo> {
irClass.getPackageFragment() as? IrExternalPackageFragment irClass.getPackageFragment() as? IrExternalPackageFragment
?: error("Expected an external package fragment for ${irClass.render()}") ?: error("Expected an external package fragment for ${irClass.render()}")
val signature = irClass.symbol.signature val signature = irClass.symbol.signature
@@ -841,8 +846,10 @@ internal class KonanIrLinker(
return serializedClassFields.fields.mapIndexed { index, field -> return serializedClassFields.fields.mapIndexed { index, field ->
if (index == serializedClassFields.outerThisIndex) { if (index == serializedClassFields.outerThisIndex) {
require(irClass.isInner) { "Expected an inner class: ${irClass.render()}" } require(irClass.isInner) { "Expected an inner class: ${irClass.render()}" }
require(outerThisField != null) { "For an inner class ${irClass.render()} there should be <outer this> field" } require(outerThisFieldInfo != null) { "For an inner class ${irClass.render()} there should be <outer this> field" }
outerThisField.toFieldInfo() outerThisFieldInfo.also {
require(it.alignment == field.alignment) { "Mismatched align information for outer this"}
}
} else { } else {
val name = fileDeserializationState.fileReader.string(field.name) val name = fileDeserializationState.fileReader.string(field.name)
val type = when { val type = when {
@@ -862,7 +869,11 @@ internal class KonanIrLinker(
} }
} }
ClassLayoutBuilder.FieldInfo( ClassLayoutBuilder.FieldInfo(
name, type, isConst = (field.flags and SerializedClassFieldInfo.FLAG_IS_CONST) != 0, irField = null) name, type,
isConst = (field.flags and SerializedClassFieldInfo.FLAG_IS_CONST) != 0,
irFieldSymbol = IrFieldSymbolImpl(),
alignment = field.alignment,
)
} }
} }
} }
@@ -99,6 +99,7 @@ using container_size_t = size_t;
// Granularity of arena container chunks. // Granularity of arena container chunks.
constexpr container_size_t kContainerAlignment = 1024; constexpr container_size_t kContainerAlignment = 1024;
// Single object alignment. // Single object alignment.
// Must match objectAlignment in Runtime.kt
constexpr container_size_t kObjectAlignment = 8; constexpr container_size_t kObjectAlignment = 8;
// Required e.g. for object size computations to be correct. // Required e.g. for object size computations to be correct.
@@ -11,6 +11,7 @@
namespace kotlin { namespace kotlin {
// Must match objectAlignment in Runtime.kt
constexpr size_t kObjectAlignment = 8; constexpr size_t kObjectAlignment = 8;
template <typename T> template <typename T>
@@ -656,6 +656,7 @@ static const TypeInfo* createTypeInfo(
result->superType_ = superType; result->superType_ = superType;
if (fieldsInfo == nullptr) { if (fieldsInfo == nullptr) {
result->instanceSize_ = superType->instanceSize_; result->instanceSize_ = superType->instanceSize_;
result->instanceAlignment_ = superType->instanceAlignment_;
result->objOffsets_ = superType->objOffsets_; result->objOffsets_ = superType->objOffsets_;
result->objOffsetsCount_ = superType->objOffsetsCount_; // So TF_IMMUTABLE can also be inherited: result->objOffsetsCount_ = superType->objOffsetsCount_; // So TF_IMMUTABLE can also be inherited:
if ((superType->flags_ & TF_IMMUTABLE) != 0) { if ((superType->flags_ & TF_IMMUTABLE) != 0) {
@@ -664,6 +665,7 @@ static const TypeInfo* createTypeInfo(
result->processObjectInMark = superType->processObjectInMark; result->processObjectInMark = superType->processObjectInMark;
} else { } else {
result->instanceSize_ = fieldsInfo->instanceSize_; result->instanceSize_ = fieldsInfo->instanceSize_;
result->instanceAlignment_ = fieldsInfo->instanceAlignment_;
result->objOffsets_ = fieldsInfo->objOffsets_; result->objOffsets_ = fieldsInfo->objOffsets_;
result->objOffsetsCount_ = fieldsInfo->objOffsetsCount_; result->objOffsetsCount_ = fieldsInfo->objOffsetsCount_;
result->processObjectInMark = fieldsInfo->processObjectInMark; result->processObjectInMark = fieldsInfo->processObjectInMark;
@@ -30,6 +30,7 @@ private:
std_support::vector<int32_t> objOffsets_; std_support::vector<int32_t> objOffsets_;
int32_t objOffsetsCount_ = 0; int32_t objOffsetsCount_ = 0;
int32_t flags_ = 0; int32_t flags_ = 0;
int32_t instanceAlignment_ = 8;
const TypeInfo* superType_ = nullptr; const TypeInfo* superType_ = nullptr;
void (*processObjectInMark_)(void*, ObjHeader*) = nullptr; void (*processObjectInMark_)(void*, ObjHeader*) = nullptr;
}; };
@@ -90,6 +91,7 @@ public:
typeInfo_.processObjectInMark = builder.processObjectInMark_; typeInfo_.processObjectInMark = builder.processObjectInMark_;
typeInfo_.flags_ = builder.flags_; typeInfo_.flags_ = builder.flags_;
typeInfo_.superType_ = builder.superType_; typeInfo_.superType_ = builder.superType_;
typeInfo_.instanceAlignment_ = builder.instanceAlignment_;
} }
TypeInfo* typeInfo() noexcept { return &typeInfo_; } TypeInfo* typeInfo() noexcept { return &typeInfo_; }
@@ -90,6 +90,10 @@ struct InterfaceTableRecord {
// This struct represents runtime type information and by itself is the compile time // This struct represents runtime type information and by itself is the compile time
// constant. // constant.
// When adding a field here do not forget to adjust:
// 1. RTTIGenerator
// 2. ObjectTestSupport TypeInfoHolder
// 3. createTypeInfo in ObjcExport.mm
struct TypeInfo { struct TypeInfo {
// Reference to self, to allow simple obtaining TypeInfo via meta-object. // Reference to self, to allow simple obtaining TypeInfo via meta-object.
const TypeInfo* typeInfo_; const TypeInfo* typeInfo_;
@@ -136,6 +140,10 @@ struct TypeInfo {
// TODO: Consider providing a generic traverse method instead. // TODO: Consider providing a generic traverse method instead.
void (*processObjectInMark)(void* state, ObjHeader* object); void (*processObjectInMark)(void* state, ObjHeader* object);
// Required alignment of instance
uint32_t instanceAlignment_;
// vtable starts just after declared contents of the TypeInfo: // vtable starts just after declared contents of the TypeInfo:
// void* const vtable_[]; // void* const vtable_[];
#ifdef __cplusplus #ifdef __cplusplus