diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/irasdescriptors/NewIrUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/irasdescriptors/NewIrUtils.kt index 7d42cf681e5..984fa5f6485 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/irasdescriptors/NewIrUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/irasdescriptors/NewIrUtils.kt @@ -114,6 +114,8 @@ val IrFunction.isSuspend get() = this is IrSimpleFunction && this.isSuspend fun IrClass.isUnit() = this.fqNameSafe == KotlinBuiltIns.FQ_NAMES.unit.toSafe() +fun IrClass.isKotlinArray() = this.fqNameSafe == KotlinBuiltIns.FQ_NAMES.array.toSafe() + fun IrClass.getSuperClassNotAny() = this.superClasses.map { it.owner }.atMostOne { !it.isInterface && !it.isAny() } fun IrClass.isAny() = this.fqNameSafe == KotlinBuiltIns.FQ_NAMES.any.toSafe() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt index d51438c08a0..a3cb46fc435 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt @@ -278,8 +278,6 @@ internal val ClassDescriptor.writableTypeInfoSymbolName: String return "ktypew:" + this.fqNameSafe.toString() } -internal val theUnitInstanceName = "kobj:kotlin.Unit" - internal val ClassDescriptor.objectInstanceFieldSymbolName: String get() { assert (this.isExported()) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 59da38f6533..42d4a5b9e29 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -272,6 +272,18 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { return function } + private fun importGlobal(name: String, otherModule: LLVMModuleRef): LLVMValueRef { + if (LLVMGetNamedGlobal(llvmModule, name) != null) { + throw IllegalArgumentException("global $name already exists") + } + + val externalGlobal = LLVMGetNamedGlobal(otherModule, name)!! + val globalType = getGlobalType(externalGlobal) + val global = LLVMAddGlobal(llvmModule, globalType, name)!! + + return global + } + private fun copyFunctionAttributes(source: LLVMValueRef, destination: LLVMValueRef) { // TODO: consider parameter attributes val attributeIndex = LLVMAttributeFunctionIndex @@ -386,6 +398,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { private fun importRtFunction(name: String) = importFunction(name, runtime.llvmModule) + private fun importRtGlobal(name: String) = importGlobal(name, runtime.llvmModule) + val allocInstanceFunction = importRtFunction("AllocInstance") val allocArrayFunction = importRtFunction("AllocArrayInstance") val initInstanceFunction = importRtFunction("InitInstance") @@ -457,6 +471,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { origin = context.standardLlvmSymbolsOrigin ) + val staticContainer = importRtGlobal("theStaticObjectsContainer") + val memsetFunction = importMemset() val usedFunctions = mutableListOf() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt index b361a8be109..9f0658ead9d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt @@ -33,10 +33,15 @@ internal fun createLlvmDeclarations(context: Context): LlvmDeclarations { context.ir.irModule.acceptChildrenVoid(generator) return with(generator) { LlvmDeclarations( - functions, classes, fields, staticFields, theUnitInstanceRef + functions, classes, fields, staticFields, uniques ) } +} +// Please note, that llvmName is part of the ABI, and cannot be liberally changed. +enum class UniqueKind(val llvmName: String) { + UNIT("theUnitInstance"), + EMPTY_ARRAY("theEmptyArray") } internal class LlvmDeclarations( @@ -44,8 +49,7 @@ internal class LlvmDeclarations( private val classes: Map, private val fields: Map, private val staticFields: Map, - private val theUnitInstanceRef: ConstPointer? -) { + private val unique: Map) { fun forFunction(descriptor: FunctionDescriptor) = functions[descriptor] ?: error(descriptor.toString()) @@ -61,7 +65,7 @@ internal class LlvmDeclarations( fun forSingleton(descriptor: ClassDescriptor) = forClass(descriptor).singletonDeclarations ?: error(descriptor.toString()) - fun getUnitInstanceRef() = theUnitInstanceRef ?: error("") + fun forUnique(kind: UniqueKind) = unique[kind] ?: error("No unique $kind") } @@ -88,6 +92,8 @@ internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMType internal class StaticFieldLlvmDeclarations(val storage: LLVMValueRef) +internal class UniqueLlvmDeclarations(val pointer: ConstPointer) + // TODO: rework getFields and getDeclaredFields. /** @@ -151,7 +157,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : val classes = mutableMapOf() val fields = mutableMapOf() val staticFields = mutableMapOf() - var theUnitInstanceRef: ConstPointer? = null + val uniques = mutableMapOf() private class Namer(val prefix: String) { private val names = mutableMapOf() @@ -266,8 +272,11 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : typeInfoPtr = typeInfoGlobal.pointer } + if (descriptor.isUnit() || descriptor.isKotlinArray()) + createUniqueDeclarations(descriptor, typeInfoPtr, bodyType) + val singletonDeclarations = if (descriptor.kind.isSingleton) { - createSingletonDeclarations(descriptor, typeInfoPtr, bodyType) + createSingletonDeclarations(descriptor) } else { null } @@ -296,14 +305,24 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : singletonDeclarations, objCDeclarations) } - private fun createSingletonDeclarations( - descriptor: ClassDescriptor, - typeInfoPtr: ConstPointer, - bodyType: LLVMTypeRef - ): SingletonLlvmDeclarations? { + private fun createUniqueDeclarations( + descriptor: ClassDescriptor, typeInfoPtr: ConstPointer, bodyType: LLVMTypeRef) { + when { + descriptor.isUnit() -> { + uniques[UniqueKind.UNIT] = + UniqueLlvmDeclarations(staticData.createUniqueInstance(UniqueKind.UNIT, bodyType, typeInfoPtr)) + } + descriptor.isKotlinArray() -> { + uniques[UniqueKind.EMPTY_ARRAY] = + UniqueLlvmDeclarations(staticData.createUniqueInstance(UniqueKind.EMPTY_ARRAY, bodyType, typeInfoPtr)) + } + else -> TODO("Unsupported unique $descriptor") + } + } + + private fun createSingletonDeclarations(descriptor: ClassDescriptor): SingletonLlvmDeclarations? { if (descriptor.isUnit()) { - this.theUnitInstanceRef = staticData.createUnitInstance(bodyType, typeInfoPtr) return null } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt index 21ba06a6b82..3281f7866bd 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt @@ -43,7 +43,6 @@ class Runtime(bitcodeFile: String) { val objHeaderType = getStructType("ObjHeader") val objHeaderPtrType = pointerType(objHeaderType) val arrayHeaderType = getStructType("ArrayHeader") - val containerHeaderType = getStructType("ContainerHeader") val frameOverlayType = getStructType("FrameOverlay") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt index 4f3757adef0..6effeeb7eb9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt @@ -31,14 +31,15 @@ import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.types.TypeProjection import org.jetbrains.kotlin.types.replace + private fun StaticData.objHeader(typeInfo: ConstPointer): Struct { - val container = NullPointer(runtime.containerHeaderType) // Static object mark. + val container = constValue(context.llvm.staticContainer) return Struct(runtime.objHeaderType, typeInfo, container) } private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct { assert (length >= 0) - val container = NullPointer(runtime.containerHeaderType) // Static object mark. + val container = constValue(context.llvm.staticContainer) return Struct(runtime.arrayHeaderType, typeInfo, container, Int32(length)) } @@ -134,26 +135,27 @@ internal fun StaticData.createArrayList(array: ConstPointer, length: Int): Const return createKotlinObject(arrayListClass, body) } - -internal fun StaticData.createUnitInstance(bodyType: LLVMTypeRef, - typeInfo: ConstPointer -): ConstPointer { +internal fun StaticData.createUniqueInstance( + kind: UniqueKind, bodyType: LLVMTypeRef, typeInfo: ConstPointer): ConstPointer { assert (getStructElements(bodyType).isEmpty()) val objHeader = objHeader(typeInfo) - val global = this.placeGlobal(theUnitInstanceName, objHeader, isExported = true) + val global = this.placeGlobal(kind.llvmName, objHeader, isExported = true) return global.pointer } -internal val ContextUtils.theUnitInstanceRef: ConstPointer - get() { - val unitDescriptor = context.ir.symbols.unit.owner - return if (isExternal(unitDescriptor)) { - constPointer(importGlobal( - theUnitInstanceName, - context.llvm.runtime.objHeaderType, - origin = unitDescriptor.llvmSymbolOrigin - )) - } else { - context.llvmDeclarations.getUnitInstanceRef() - } +internal fun ContextUtils.unique(kind: UniqueKind): ConstPointer { + val descriptor = when (kind) { + UniqueKind.UNIT -> context.ir.symbols.unit.owner + UniqueKind.EMPTY_ARRAY -> context.ir.symbols.array.owner } + return if (isExternal(descriptor)) { + constPointer(importGlobal( + kind.llvmName, context.llvm.runtime.objHeaderType, origin = descriptor.llvmSymbolOrigin + )) + } else { + context.llvmDeclarations.forUnique(kind).pointer + } +} + +internal val ContextUtils.theUnitInstanceRef: ConstPointer + get() = this.unique(UniqueKind.UNIT) \ No newline at end of file diff --git a/runtime/src/main/cpp/Arrays.cpp b/runtime/src/main/cpp/Arrays.cpp index 8fc6418cab6..9937125ed32 100644 --- a/runtime/src/main/cpp/Arrays.cpp +++ b/runtime/src/main/cpp/Arrays.cpp @@ -25,10 +25,6 @@ namespace { -const ArrayHeader anEmptyArray = { - const_cast(theArrayTypeInfo), /* permanent object */ 0, /* element count */ 0 -}; - ALWAYS_INLINE inline void mutabilityCheck(KConstRef thiz) { // TODO: optimize it! if (thiz->container()->frozen()) { @@ -56,6 +52,9 @@ inline void copyImpl(KConstRef thiz, KInt fromIndex, extern "C" { +// Generated as part of Kotlin standard library. +extern const ObjHeader theEmptyArray; + // TODO: those must be compiler intrinsics afterwards. // Array.kt @@ -117,7 +116,7 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex, // Arrays.kt OBJ_GETTER0(Kotlin_emptyArray) { - RETURN_OBJ(const_cast(anEmptyArray.obj())); + RETURN_OBJ(const_cast(&theEmptyArray)); } KByte Kotlin_ByteArray_get(KConstRef thiz, KInt index) { diff --git a/runtime/src/main/cpp/Assert.h b/runtime/src/main/cpp/Assert.h index bacac916426..50e88dfce57 100644 --- a/runtime/src/main/cpp/Assert.h +++ b/runtime/src/main/cpp/Assert.h @@ -19,14 +19,29 @@ #include "Common.h" -RUNTIME_NORETURN void RuntimeAssertFailed(const char* location, const char* message); +// To avoid cluttering optimized code with asserts, they could be turned off. +#define KONAN_ENABLE_ASSERT 1 #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) +RUNTIME_NORETURN void RuntimeAssertFailed(const char* location, const char* message); + +#if KONAN_ENABLE_ASSERT +// Use RuntimeAssert() in internal state checks, which could be ignored in production. #define RuntimeAssert(condition, message) \ if (!(condition)) { \ RuntimeAssertFailed( __FILE__ ":" TOSTRING(__LINE__), message); \ } +#else +#define RuntimeAssert(condition, message) +#endif + +// Use RuntimeCheck() in runtime checks that could fail due to external condition and shall lead +// to program termination. Never compiled out. +#define RuntimeCheck(condition, message) \ + if (!(condition)) { \ + RuntimeAssertFailed( __FILE__ ":" TOSTRING(__LINE__), message); \ + } #endif // RUNTIME_ASSERT_H diff --git a/runtime/src/main/cpp/Exceptions.cpp b/runtime/src/main/cpp/Exceptions.cpp index ebb68797f17..47daa8c7457 100644 --- a/runtime/src/main/cpp/Exceptions.cpp +++ b/runtime/src/main/cpp/Exceptions.cpp @@ -63,14 +63,14 @@ struct Backtrace { auto result = AllocArrayInstance( theArrayTypeInfo, count - skipCount, arrayHolder.slot()); // TODO: throw cached OOME? - RuntimeAssert(result != nullptr, "Cannot create backtrace array"); + RuntimeCheck(result != nullptr, "Cannot create backtrace array"); } void setNextElement(const char* element) { auto result = CreateStringFromCString( element, ArrayAddressOfElementAt(obj()->array(), index++)); // TODO: throw cached OOME? - RuntimeAssert(result != nullptr, "Cannot create backtrace array element"); + RuntimeCheck(result != nullptr, "Cannot create backtrace array element"); } ObjHeader* obj() { return arrayHolder.obj(); } @@ -144,7 +144,7 @@ OBJ_GETTER0(GetCurrentStackTrace) { int size = backtrace(buffer, maxSize); char** symbols = backtrace_symbols(buffer, size); - RuntimeAssert(symbols != nullptr, "Not enough memory to retrieve the stacktrace"); + RuntimeCheck(symbols != nullptr, "Not enough memory to retrieve the stacktrace"); if (size < kSkipFrames) return AllocArrayInstance(theArrayTypeInfo, 0, OBJ_RESULT); AutoFree autoFree(symbols); @@ -166,7 +166,7 @@ void ThrowException(KRef exception) { "Throwing something non-throwable"); #if KONAN_NO_EXCEPTIONS PrintThrowable(exception); - RuntimeAssert(false, "Exceptions unsupported"); + RuntimeCheck(false, "Exceptions unsupported"); #else throw ObjHolder(exception); #endif diff --git a/runtime/src/main/cpp/JSInterop.cpp b/runtime/src/main/cpp/JSInterop.cpp index e45c459d7c3..85a37ae4d8d 100644 --- a/runtime/src/main/cpp/JSInterop.cpp +++ b/runtime/src/main/cpp/JSInterop.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "Porting.h" #include "Types.h" typedef KInt Arena; @@ -27,14 +28,17 @@ extern "C" { // These functions are implemented in JS file for WASM and are not available on other platforms. RUNTIME_NORETURN Arena Konan_js_allocateArena() { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN void Konan_js_freeArena(Arena arena) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN void Konan_js_pushIntToArena(Arena arena, KInt value) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN KInt Konan_js_getInt(Arena arena, @@ -42,6 +46,7 @@ RUNTIME_NORETURN KInt Konan_js_getInt(Arena arena, Pointer propertyPtr, KInt propertyLen) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN KInt Konan_js_getProperty(Arena arena, @@ -49,6 +54,7 @@ RUNTIME_NORETURN KInt Konan_js_getProperty(Arena arena, Pointer propertyPtr, KInt propertyLen) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN void Konan_js_setFunction(Arena arena, @@ -57,6 +63,7 @@ RUNTIME_NORETURN void Konan_js_setFunction(Arena arena, KInt propertyLength, KInt function) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } RUNTIME_NORETURN void Konan_js_setString(Arena arena, @@ -66,6 +73,7 @@ RUNTIME_NORETURN void Konan_js_setString(Arena arena, Pointer stringPtr, KInt stringLength) { RuntimeAssert(false, "JavaScript interop is disabled"); + konan::abort(); } }; // extern "C" diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 4418ef03539..1f69345d592 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -39,12 +39,6 @@ // Auto-adjust GC thresholds. #define GC_ERGONOMICS 1 -// TODO: ensure it is read-only. -ContainerHeader ObjHeader::theStaticObjectsContainer = { - CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT, - 0 /* Object count */ -}; - namespace { // Granularity of arena container chunks. @@ -407,6 +401,13 @@ inline bool isRefCounted(KConstRef object) { extern "C" { +// Ensure LLVM never throws theStaticObjectsContainer away. +// TODO: although practically const, marking it as such makes LLVM crazy, fix it. +RUNTIME_USED ContainerHeader theStaticObjectsContainer = { + CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT, + 0 /* Object count */ +}; + void objc_release(void* ptr); void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject); RUNTIME_NORETURN void ThrowFreezingException(); diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 99f0254ba24..7e6a3fa52bf 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -200,10 +200,8 @@ struct ObjHeader { reinterpret_cast(typeInfoOrMeta_) : createMetaObject(&typeInfoOrMeta_); } - static ContainerHeader theStaticObjectsContainer; - ContainerHeader* container() const { - return container_ == nullptr ? &theStaticObjectsContainer : container_; + return container_; } // Unsafe cast to ArrayHeader. Use carefully! diff --git a/runtime/src/main/cpp/Porting.cpp b/runtime/src/main/cpp/Porting.cpp index 02e6e5b0736..f352473fa7a 100644 --- a/runtime/src/main/cpp/Porting.cpp +++ b/runtime/src/main/cpp/Porting.cpp @@ -36,8 +36,8 @@ #include "Porting.h" #if KONAN_WASM || KONAN_ZEPHYR -extern "C" void Konan_abort(const char*); -extern "C" void Konan_exit(int32_t status); +extern "C" RUNTIME_NORETURN void Konan_abort(const char*); +extern "C" RUNTIME_NORETURN void Konan_exit(int32_t status); #endif #ifdef KONAN_ZEPHYR // In Zephyr's Newlib strnlen(3) is not included from string.h by default. diff --git a/runtime/src/main/cpp/Porting.h b/runtime/src/main/cpp/Porting.h index 3ed6b55dd52..b66318c3627 100644 --- a/runtime/src/main/cpp/Porting.h +++ b/runtime/src/main/cpp/Porting.h @@ -33,8 +33,8 @@ void consoleErrorUtf8(const void* utf8, uint32_t sizeBytes); int32_t consoleReadUtf8(void* utf8, uint32_t maxSizeBytes); // Process control. -void abort(void); -void exit(int32_t status); +RUNTIME_NORETURN void abort(void); +RUNTIME_NORETURN void exit(int32_t status); // Thread control. void onThreadExit(void (*destructor)());