diff --git a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/Pinning.kt b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/Pinning.kt index 8f5a3e2f1fb..eabb6b044de 100644 --- a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/Pinning.kt +++ b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/Pinning.kt @@ -44,16 +44,16 @@ inline fun T.usePinned(block: (Pinned) -> R): R { } } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun ByteArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun ShortArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun IntArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun LongArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } // TODO: pinning of unsigned arrays involves boxing as they are inline classes wrapping signed arrays. @@ -69,10 +69,10 @@ fun UIntArray.refTo(index: Int): CValuesRef = this.usingPinned { addres fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun ULongArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun FloatArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } -fun Pinned.addressOf(index: Int): CPointer = this.addressOfElement(index) +fun Pinned.addressOf(index: Int): CPointer = this.get().addressOfElement(index) fun DoubleArray.refTo(index: Int): CValuesRef = this.usingPinned { addressOf(index) } private inline fun T.usingPinned( @@ -86,21 +86,32 @@ private inline fun T.usingPinned( } } -@SymbolName("Kotlin_Arrays_getAddressOfElement") -private external fun getAddressOfElement(array: Any, index: Int): COpaquePointer +@SymbolName("Kotlin_Arrays_getByteArrayAddressOfElement") +private external fun ByteArray.addressOfElement(index: Int): CPointer -@Suppress("NOTHING_TO_INLINE") -private inline fun

Pinned<*>.addressOfElement(index: Int): CPointer

= - getAddressOfElement(this.get(), index).reinterpret() +@SymbolName("Kotlin_Arrays_getShortArrayAddressOfElement") +private external fun ShortArray.addressOfElement(index: Int): CPointer -@SymbolName("Kotlin_Arrays_getAddressOfElement") +@SymbolName("Kotlin_Arrays_getIntArrayAddressOfElement") +private external fun IntArray.addressOfElement(index: Int): CPointer + +@SymbolName("Kotlin_Arrays_getLongArrayAddressOfElement") +private external fun LongArray.addressOfElement(index: Int): CPointer + +@SymbolName("Kotlin_Arrays_getByteArrayAddressOfElement") private external fun UByteArray.addressOfElement(index: Int): CPointer -@SymbolName("Kotlin_Arrays_getAddressOfElement") +@SymbolName("Kotlin_Arrays_getShortArrayAddressOfElement") private external fun UShortArray.addressOfElement(index: Int): CPointer -@SymbolName("Kotlin_Arrays_getAddressOfElement") +@SymbolName("Kotlin_Arrays_getIntArrayAddressOfElement") private external fun UIntArray.addressOfElement(index: Int): CPointer -@SymbolName("Kotlin_Arrays_getAddressOfElement") +@SymbolName("Kotlin_Arrays_getLongArrayAddressOfElement") private external fun ULongArray.addressOfElement(index: Int): CPointer + +@SymbolName("Kotlin_Arrays_getFloatArrayAddressOfElement") +private external fun FloatArray.addressOfElement(index: Int): CPointer + +@SymbolName("Kotlin_Arrays_getDoubleArrayAddressOfElement") +private external fun DoubleArray.addressOfElement(index: Int): CPointer diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 6e1aa1f363d..f1757ee8b0d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -1592,7 +1592,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map): LLVMTypeRef { - val fieldTypes = fields.map { getLLVMType(it.type) } + val fieldTypes = listOf(runtime.objHeaderType) + fields.map { getLLVMType(it.type) } + // TODO: consider adding synthetic ObjHeader field to Any. val classType = LLVMStructCreateNamed(LLVMGetModuleContext(context.llvmModule), name)!! @@ -369,7 +370,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : error(containingClass.descriptor.toString()) val allFields = classDeclarations.fields this.fields[descriptor] = FieldLlvmDeclarations( - allFields.indexOf(descriptor), + allFields.indexOf(descriptor) + 1, // First field is ObjHeader. classDeclarations.bodyType ) } else { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt index 127c63f0ef6..d125617b6b9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt @@ -289,9 +289,9 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { NullPointer(int32Type), NullPointer(int8Type), NullPointer(kInt8Ptr)) } else { data class FieldRecord(val offset: Int, val type: Int, val name: String) - val fields = getStructElements(bodyType).mapIndexedNotNull { index, type -> + val fields = getStructElements(bodyType).drop(1).mapIndexedNotNull { index, type -> FieldRecord( - LLVMOffsetOfElement(llvmTargetData, bodyType, index).toInt(), + LLVMOffsetOfElement(llvmTargetData, bodyType, index + 1).toInt(), mapRuntimeType(type), llvmDeclarations.fields[index].name.asString()) } @@ -316,7 +316,7 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { ): ConstPointer { assert(descriptor.isInterface) - val size = 0 + val size = LLVMStoreSizeOfType(llvmTargetData, kObjHeader).toInt() val superClass = context.ir.symbols.any.owner diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt index 001fbd5c22b..9f22b0c2189 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt @@ -71,24 +71,20 @@ internal fun StaticData.createConstKotlinArray(arrayClass: IrClass, elements: Li return createRef(objHeaderPtr) } -internal fun StaticData.createConstKotlinObject(type: IrClass, body: ConstValue): ConstPointer { +internal fun StaticData.createConstKotlinObject(type: IrClass, vararg fields: ConstValue): ConstPointer { val typeInfo = type.typeInfoPtr - - val compositeType = structType(runtime.objHeaderType, body.llvmType) - - val global = this.createGlobal(compositeType, "") - - val objHeaderPtr = global.pointer.getElementPtr(0) val objHeader = objHeader(typeInfo) - global.setInitializer(Struct(compositeType, objHeader, body)) + val global = this.placeGlobal("", Struct(objHeader, *fields)) global.setConstant(true) + val objHeaderPtr = global.pointer.getElementPtr(0) + return createRef(objHeaderPtr) } -internal fun StaticData.createInitializer(type: IrClass, body: ConstValue): ConstValue = - Struct(objHeader(type.typeInfoPtr), body) +internal fun StaticData.createInitializer(type: IrClass, vararg fields: ConstValue): ConstValue = + Struct(objHeader(type.typeInfoPtr), *fields) private fun StaticData.getArrayListClass(): ClassDescriptor { val module = context.irModule!!.descriptor @@ -122,15 +118,13 @@ internal fun StaticData.createConstArrayList(array: ConstPointer, length: Int): val fqName = it.fqNameSafe.asString() sorted.put(fqName, arrayListFields[fqName]!!) } - - val body = Struct(*(sorted.values.toTypedArray())) - return createConstKotlinObject(arrayListClass, body) + return createConstKotlinObject(arrayListClass, *sorted.values.toTypedArray()) } internal fun StaticData.createUniqueInstance( kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer { - assert (getStructElements(bodyType).isEmpty()) + assert (getStructElements(bodyType).size == 1) // ObjHeader only. val objHeader = when (kind) { UniqueKind.UNIT -> objHeader(typeInfo) UniqueKind.EMPTY_ARRAY -> arrayHeader(typeInfo, 0) diff --git a/runtime/src/main/cpp/Arrays.cpp b/runtime/src/main/cpp/Arrays.cpp index 91546c7704b..440678eaa6b 100644 --- a/runtime/src/main/cpp/Arrays.cpp +++ b/runtime/src/main/cpp/Arrays.cpp @@ -489,13 +489,58 @@ KNativePtr Kotlin_ImmutableBlob_asCPointerImpl(KRef thiz, KInt offset) { return PrimitiveArrayAddressOfElementAt(array, offset); } -KNativePtr Kotlin_Arrays_getAddressOfElement(KRef thiz, KInt index) { +KNativePtr Kotlin_Arrays_getByteArrayAddressOfElement(KRef thiz, KInt index) { ArrayHeader* array = thiz->array(); if (index < 0 || index >= array->count_) { ThrowArrayIndexOutOfBoundsException(); } - return AddressOfElementAt(array, index); + return AddressOfElementAt(array, index); +} + +KNativePtr Kotlin_Arrays_getShortArrayAddressOfElement(KRef thiz, KInt index) { + ArrayHeader* array = thiz->array(); + if (index < 0 || index >= array->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + + return AddressOfElementAt(array, index); +} + +KNativePtr Kotlin_Arrays_getIntArrayAddressOfElement(KRef thiz, KInt index) { + ArrayHeader* array = thiz->array(); + if (index < 0 || index >= array->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + + return AddressOfElementAt(array, index); +} + +KNativePtr Kotlin_Arrays_getLongArrayAddressOfElement(KRef thiz, KInt index) { + ArrayHeader* array = thiz->array(); + if (index < 0 || index >= array->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + + return AddressOfElementAt(array, index); +} + +KNativePtr Kotlin_Arrays_getFloatArrayAddressOfElement(KRef thiz, KInt index) { + ArrayHeader* array = thiz->array(); + if (index < 0 || index >= array->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + + return AddressOfElementAt(array, index); +} + +KNativePtr Kotlin_Arrays_getDoubleArrayAddressOfElement(KRef thiz, KInt index) { + ArrayHeader* array = thiz->array(); + if (index < 0 || index >= array->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + + return AddressOfElementAt(array, index); } } // extern "C" diff --git a/runtime/src/main/cpp/Atomic.cpp b/runtime/src/main/cpp/Atomic.cpp index c10f342b913..add924bc1ab 100644 --- a/runtime/src/main/cpp/Atomic.cpp +++ b/runtime/src/main/cpp/Atomic.cpp @@ -23,37 +23,48 @@ namespace { struct AtomicReferenceLayout { + ObjHeader header; KRef value_; KInt lock_; }; +template struct AtomicPrimitive { + ObjHeader header; + volatile T value_; +}; + +template inline volatile T* getValueLocation(KRef thiz) { + AtomicPrimitive* atomic = reinterpret_cast*>(thiz); + return &atomic->value_; +} + template void setImpl(KRef thiz, T value) { - volatile T* location = reinterpret_cast(thiz + 1); + volatile T* location = getValueLocation(thiz); atomicSet(location, value); } template T getImpl(KRef thiz) { - volatile T* location = reinterpret_cast(thiz + 1); + volatile T* location = getValueLocation(thiz); return atomicGet(location); } template T addAndGetImpl(KRef thiz, T delta) { - volatile T* location = reinterpret_cast(thiz + 1); + volatile T* location = getValueLocation(thiz); return atomicAdd(location, delta); } template T compareAndSwapImpl(KRef thiz, T expectedValue, T newValue) { - volatile T* location = reinterpret_cast(thiz + 1); - return compareAndSwap(location, expectedValue, newValue); + volatile T* location = getValueLocation(thiz); + return compareAndSwap(location, expectedValue, newValue); } template KBoolean compareAndSetImpl(KRef thiz, T expectedValue, T newValue) { - volatile T* location = reinterpret_cast(thiz + 1); + volatile T* location = getValueLocation(thiz); return compareAndSet(location, expectedValue, newValue); } inline AtomicReferenceLayout* asAtomicReference(KRef thiz) { - return reinterpret_cast(thiz + 1); + return reinterpret_cast(thiz); } } // namespace @@ -93,7 +104,7 @@ KLong Kotlin_AtomicLong_compareAndSwap(KRef thiz, KLong expectedValue, KLong new // Potentially huge performance penalty, but correct. // TODO: reconsider, once target MIPS can do proper 64-bit CAS. while (compareAndSwap(&lock64, 0, 1) != 0); - KLong* address = reinterpret_cast(thiz + 1); + volatile KLong* address = getValueLocation(thiz); KLong old = *address; if (old == expectedValue) { *address = newValue; @@ -111,7 +122,7 @@ KBoolean Kotlin_AtomicLong_compareAndSet(KRef thiz, KLong expectedValue, KLong n // TODO: reconsider, once target MIPS can do proper 64-bit CAS. KBoolean result = false; while (compareAndSwap(&lock64, 0, 1) != 0); - KLong* address = reinterpret_cast(thiz + 1); + volatile KLong* address = getValueLocation(thiz); KLong old = *address; if (old == expectedValue) { result = true; @@ -129,7 +140,7 @@ void Kotlin_AtomicLong_set(KRef thiz, KLong newValue) { // Potentially huge performance penalty, but correct. // TODO: reconsider, once target MIPS can do proper 64-bit atomic store. while (compareAndSwap(&lock64, 0, 1) != 0); - KLong* address = reinterpret_cast(thiz + 1); + volatile KLong* address = getValueLocation(thiz); *address = newValue; compareAndSwap(&lock64, 1, 0); #else @@ -142,7 +153,7 @@ KLong Kotlin_AtomicLong_get(KRef thiz) { // Potentially huge performance penalty, but correct. // TODO: reconsider, once target MIPS can do proper 64-bit atomic store. while (compareAndSwap(&lock64, 0, 1) != 0); - KLong* address = reinterpret_cast(thiz + 1); + volatile KLong* address = getValueLocation(thiz); KLong value = *address; compareAndSwap(&lock64, 1, 0); return value; diff --git a/runtime/src/main/cpp/KDebug.cpp b/runtime/src/main/cpp/KDebug.cpp index be48041d532..a9b669f377b 100644 --- a/runtime/src/main/cpp/KDebug.cpp +++ b/runtime/src/main/cpp/KDebug.cpp @@ -45,6 +45,19 @@ constexpr int runtimeTypeSize[] = { 1 // BOOLEAN }; +constexpr int runtimeTypeAlignment[] = { + -1, // INVALID + alignof(ObjHeader*), // OBJECT + alignof(int8_t), // INT8 + alignof(int16_t), // INT16 + alignof(int32_t), // INT32 + alignof(int64_t), // INT64 + alignof(float), // FLOAT32 + alignof(double), // FLOAT64 + alignof(void*), // NATIVE_PTR + 1 // BOOLEAN +}; + } // namespace extern "C" { @@ -130,13 +143,16 @@ RUNTIME_USED void* Konan_DebugGetFieldAddress(KRef obj, int index) { if (index > obj->array()->count_) return nullptr; - return reinterpret_cast(obj->array() + 1) + index * runtimeTypeSize[-extendedTypeInfo->fieldsCount_]; + int32_t typeIndex = -extendedTypeInfo->fieldsCount_; + return reinterpret_cast(obj->array()) + + alignUp(sizeof(struct ArrayHeader), runtimeTypeAlignment[typeIndex]) + + index * runtimeTypeSize[typeIndex]; } if (index >= extendedTypeInfo->fieldsCount_) return nullptr; - return reinterpret_cast(obj + 1) + extendedTypeInfo->fieldOffsets_[index]; + return reinterpret_cast(obj) + extendedTypeInfo->fieldOffsets_[index]; } // Compute address of field or an array element at the index, or null, if incorrect. diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index c6091aecb3b..19ef63e108b 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -47,6 +47,9 @@ constexpr container_size_t kContainerAlignment = 1024; // Single object alignment. constexpr container_size_t kObjectAlignment = 8; +// Required e.g. for object size computations to be correct. +static_assert(sizeof(ContainerHeader) % kObjectAlignment == 0, "sizeof(ContainerHeader) is not aligned"); + #if TRACE_MEMORY #define MEMORY_LOG(...) konan::consolePrintf(__VA_ARGS__); #else @@ -396,14 +399,25 @@ inline container_size_t alignUp(container_size_t size, int alignment) { return (size + alignment - 1) & ~(alignment - 1); } +inline uint32_t arrayObjectSize(const TypeInfo* typeInfo, uint32_t count) { + // Note: array body is aligned, but for size computation it is enough to align the sum. + static_assert(kObjectAlignment % alignof(KLong) == 0, ""); + static_assert(kObjectAlignment % alignof(KDouble) == 0, ""); + return alignUp(sizeof(ArrayHeader) - typeInfo->instanceSize_ * count, kObjectAlignment); +} + +inline uint32_t arrayObjectSize(const ArrayHeader* obj) { + return arrayObjectSize(obj->type_info(), obj->count_); +} + // TODO: shall we do padding for alignment? inline container_size_t objectSize(const ObjHeader* obj) { const TypeInfo* type_info = obj->type_info(); container_size_t size = (type_info->instanceSize_ < 0 ? // An array. - ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader) + arrayObjectSize(obj->array()) : - type_info->instanceSize_ + sizeof(ObjHeader)); + type_info->instanceSize_); return alignUp(size, kObjectAlignment); } @@ -494,7 +508,7 @@ inline void traverseContainerObjectFields(ContainerHeader* container, func proce if (typeInfo != theArrayTypeInfo) { for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { ObjHeader** location = reinterpret_cast( - reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); + reinterpret_cast(obj) + typeInfo->objOffsets_[index]); process(location); } } else { @@ -1003,7 +1017,7 @@ void FreeContainer(ContainerHeader* container) { void ObjectContainer::Init(const TypeInfo* typeInfo) { RuntimeAssert(typeInfo->instanceSize_ >= 0, "Must be an object"); uint32_t alloc_size = - sizeof(ContainerHeader) + sizeof(ObjHeader) + typeInfo->instanceSize_; + sizeof(ContainerHeader) + typeInfo->instanceSize_; header_ = AllocContainer(alloc_size); if (header_) { // One object in this container. @@ -1018,8 +1032,7 @@ void ObjectContainer::Init(const TypeInfo* typeInfo) { void ArrayContainer::Init(const TypeInfo* typeInfo, uint32_t elements) { RuntimeAssert(typeInfo->instanceSize_ < 0, "Must be an array"); uint32_t alloc_size = - sizeof(ContainerHeader) + sizeof(ArrayHeader) - - typeInfo->instanceSize_ * elements; + sizeof(ContainerHeader) + arrayObjectSize(typeInfo, elements); header_ = AllocContainer(alloc_size); RuntimeAssert(header_ != nullptr, "Cannot alloc memory"); if (header_) { @@ -1030,7 +1043,7 @@ void ArrayContainer::Init(const TypeInfo* typeInfo, uint32_t elements) { SetHeader(GetPlace()->obj(), typeInfo); MEMORY_LOG("array at %p\n", GetPlace()) OBJECT_ALLOC_EVENT( - memoryState, -typeInfo->instanceSize_ * elements, GetPlace()->obj()) + memoryState, arrayObjectSize(typeInfo, elements), GetPlace()->obj()) } } @@ -1102,7 +1115,7 @@ ObjHeader** ArenaContainer::getSlot() { ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader); + uint32_t size = type_info->instanceSize_; ObjHeader* result = reinterpret_cast(place(size)); if (!result) { return nullptr; @@ -1115,12 +1128,12 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t count) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); - container_size_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count; + container_size_t size = arrayObjectSize(type_info, count); ArrayHeader* result = reinterpret_cast(place(size)); if (!result) { return nullptr; } - OBJECT_ALLOC_EVENT(memoryState, -type_info->instanceSize_ * count, result->obj()) + OBJECT_ALLOC_EVENT(memoryState, arrayObjectSize(type_info, count), result->obj()) currentChunk_->asHeader()->incObjectCount(); setHeader(result->obj(), type_info); result->count_ = count; @@ -1923,7 +1936,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what if (!acyclic) return false; } UpdateRef(reinterpret_cast( - reinterpret_cast(where + 1) + where->type_info()->objOffsets_[index]), what); + reinterpret_cast(where) + where->type_info()->objOffsets_[index]), what); // Fence on updated location? return true; } diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 0163ff00d67..e80c37ada40 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -334,11 +334,6 @@ inline bool PermanentOrFrozen(ObjHeader* obj) { return container == nullptr || container->frozen(); } -inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { - // Instance size is negative. - return -obj->type_info()->instanceSize_ * obj->count_; -} - // Class representing arbitrary placement container. class Container { protected: diff --git a/runtime/src/main/cpp/Natives.h b/runtime/src/main/cpp/Natives.h index f6e30427fc6..467613c1755 100644 --- a/runtime/src/main/cpp/Natives.h +++ b/runtime/src/main/cpp/Natives.h @@ -20,60 +20,64 @@ #include "Types.h" #include "Exceptions.h" -inline void* AddressOfElementAt(ArrayHeader* obj, int32_t index) { - // Instance size is negative. - return reinterpret_cast(obj + 1) - - obj->type_info()->instanceSize_ * index; +constexpr size_t alignUp(size_t size, size_t alignment) { + return (size + alignment - 1) & ~(alignment - 1); } -inline const void* AddressOfElementAt(const ArrayHeader* obj, int32_t index) { - // Instance size is negative. - return reinterpret_cast(obj + 1) - - obj->type_info()->instanceSize_ * index; +template +inline T* AddressOfElementAt(ArrayHeader* obj, KInt index) { + int8_t* body = reinterpret_cast(obj) + alignUp(sizeof(ArrayHeader), alignof(T)); + return reinterpret_cast(body) + index; +} + +template +inline const T* AddressOfElementAt(const ArrayHeader* obj, KInt index) { + const int8_t* body = reinterpret_cast(obj) + alignUp(sizeof(ArrayHeader), alignof(T)); + return reinterpret_cast(body) + index; } // Optimized versions not accessing type info. inline KByte* ByteArrayAddressOfElementAt(ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline const KByte* ByteArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline KChar* CharArrayAddressOfElementAt(ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline const KChar* CharArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline KInt* IntArrayAddressOfElementAt(ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline const KInt* IntArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } // Consider aligning of base to sizeof(T). template inline T* PrimitiveArrayAddressOfElementAt(ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } template inline const T* PrimitiveArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline KRef* ArrayAddressOfElementAt(ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } inline const KRef* ArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { - return reinterpret_cast(obj + 1) + index; + return AddressOfElementAt(obj, index); } #ifdef __cplusplus diff --git a/runtime/src/main/cpp/Weak.cpp b/runtime/src/main/cpp/Weak.cpp index 0fd3e4c1ea8..4c5d679249c 100644 --- a/runtime/src/main/cpp/Weak.cpp +++ b/runtime/src/main/cpp/Weak.cpp @@ -18,7 +18,17 @@ namespace { -// TODO: an ugly hack with fixed offsets. +// TODO: an ugly hack with fixed layout. +struct WeakReferenceCounter { + ObjHeader header; + KRef referred; + KInt lock; +}; + +inline WeakReferenceCounter* asWeakReferenceCounter(ObjHeader* obj) { + return reinterpret_cast(obj); +} + constexpr int referredOffset = 0; constexpr int lockOffset = sizeof(void*); @@ -64,11 +74,11 @@ OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) { // Materialize a weak reference to either null or the real reference. OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) { - ObjHeader** referredAddress = reinterpret_cast(reinterpret_cast(counter + 1) + referredOffset); + ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred; #if KONAN_NO_THREADS RETURN_OBJ(*referredAddress); #else - int32_t* lockAddress = reinterpret_cast(reinterpret_cast(counter + 1) + lockOffset); + int32_t* lockAddress = &asWeakReferenceCounter(counter)->lock; // Spinlock. lock(lockAddress); ObjHolder holder(*referredAddress); @@ -78,12 +88,12 @@ OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) { } void WeakReferenceCounterClear(ObjHeader* counter) { - ObjHeader** referredAddress = reinterpret_cast(reinterpret_cast(counter + 1) + referredOffset); + ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred; // Note, that we don't do UpdateRef here, as reference is weak. #if KONAN_NO_THREADS *referredAddress = nullptr; #else - int32_t* lockAddress = reinterpret_cast(reinterpret_cast(counter + 1) + lockOffset); + int32_t* lockAddress = &asWeakReferenceCounter(counter)->lock; // Spinlock. lock(lockAddress); *referredAddress = nullptr;