diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt index eb4e40a6918..75dff7d1e38 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt @@ -21,6 +21,8 @@ import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrEnumEntry import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.util.SymbolTable +import org.jetbrains.kotlin.ir.util.isEnumClass +import org.jetbrains.kotlin.ir.util.isEnumEntry import org.jetbrains.kotlin.ir.util.referenceFunction import org.jetbrains.kotlin.konan.target.* import org.jetbrains.kotlin.name.isChildOf diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index a77f12aa995..2c44887afed 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -348,9 +348,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { val coverage = CoverageManager(this) - // Cache used for source offset->(line,column) mapping. - val fileEntryCache = mutableMapOf() - protected fun separator(title: String) { println("\n\n--- ${title} ----------------------\n") } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt index 4620be97fc2..101c7a9848b 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt @@ -129,22 +129,13 @@ internal val IrClass.writableTypeInfoSymbolName: String return "ktypew:" + this.fqNameForIrSerialization.toString() } -internal val IrClass.objectInstanceFieldSymbolName: String +internal val IrClass.objectInstanceGetterSymbolName: String get() { assert (this.isExported()) assert (this.kind.isSingleton) assert (!this.isUnit()) - return "kobjref:$fqNameForIrSerialization" - } - -internal val IrClass.objectInstanceShadowFieldSymbolName: String - get() { - assert (this.isExported()) - assert (this.kind.isSingleton) - assert (!this.isUnit()) - - return "kshadowobjref:$fqNameForIrSerialization" + return "kobjget:$fqNameForIrSerialization" } val IrFunction.functionName get() = with(KonanMangler) { functionName } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index dfc2eb92df5..0c0647e4bd5 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -64,46 +64,6 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun functionEntryPointAddress(function: IrFunction) = function.entryPointAddress.llvm fun functionHash(function: IrFunction): LLVMValueRef = function.functionName.localHash.llvm - fun getObjectInstanceStorage(irClass: IrClass, kind: ObjectStorageKind): LLVMValueRef { - assert (!irClass.isUnit()) - val llvmGlobal = if (!isExternal(irClass)) { - context.llvmDeclarations.forSingleton(irClass).instanceFieldRef - } else { - val llvmType = getLLVMType(irClass.defaultType) - importGlobal( - irClass.objectInstanceFieldSymbolName, - llvmType, - origin = irClass.llvmSymbolOrigin, - threadLocal = kind == ObjectStorageKind.THREAD_LOCAL - ) - } - when (kind) { - ObjectStorageKind.SHARED -> context.llvm.sharedObjects += llvmGlobal - ObjectStorageKind.THREAD_LOCAL -> context.llvm.objects += llvmGlobal - ObjectStorageKind.PERMANENT -> { /* Do nothing, no need to free such an instance. */ } - } - - return llvmGlobal - } - - fun getObjectInstanceShadowStorage(irClass: IrClass): LLVMValueRef { - assert (!irClass.isUnit()) - assert (irClass.storageKind(context) == ObjectStorageKind.SHARED) - val llvmGlobal = if (!isExternal(irClass)) { - context.llvmDeclarations.forSingleton(irClass).instanceShadowFieldRef!! - } else { - val llvmType = getLLVMType(irClass.defaultType) - importGlobal( - irClass.objectInstanceShadowFieldSymbolName, - llvmType, - origin = irClass.llvmSymbolOrigin, - threadLocal = true - ) - } - context.llvm.objects += llvmGlobal - return llvmGlobal - } - fun typeInfoForAllocation(constructedClass: IrClass): LLVMValueRef { assert(!constructedClass.isObjCClass()) return typeInfoValue(constructedClass) @@ -911,11 +871,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, return null } - fun getObjectValue( - irClass: IrClass, - exceptionHandler: ExceptionHandler, - startLocationInfo: LocationInfo?, - endLocationInfo: LocationInfo? = null + fun getObjectValue(irClass: IrClass, exceptionHandler: ExceptionHandler, + startLocationInfo: LocationInfo?, endLocationInfo: LocationInfo? = null ): LLVMValueRef { // TODO: could be processed the same way as other stateless objects. if (irClass.isUnit()) { @@ -926,7 +883,6 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, val parent = irClass.parent as IrClass if (parent.isObjCClass()) { // TODO: cache it too. - return call( codegen.llvmFunction(context.ir.symbols.interopInterpretObjCPointer.owner), listOf(getObjCClass(parent, exceptionHandler)), @@ -935,12 +891,33 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, ) } } - val storageKind = irClass.storageKind(context) - if (storageKind == ObjectStorageKind.PERMANENT) { - return loadSlot(codegen.getObjectInstanceStorage(irClass, storageKind), false) + + // If object is imported - access it via getter function. + if (isExternal(irClass)) { + val valueGetterName = irClass.objectInstanceGetterSymbolName + val valueGetterFunction = LLVMGetNamedFunction(context.llvmModule, valueGetterName) ?: + LLVMAddFunction(context.llvmModule, valueGetterName, + functionType(kObjHeaderPtr, false, kObjHeaderPtrPtr)) + return call(valueGetterFunction!!, + listOf(), + resultLifetime = Lifetime.GLOBAL, + exceptionHandler = exceptionHandler) } - val objectPtr = codegen.getObjectInstanceStorage(irClass, storageKind) + val storageKind = irClass.storageKind(context) + when (storageKind) { + ObjectStorageKind.SHARED -> context.llvm.sharedObjects += irClass + ObjectStorageKind.THREAD_LOCAL -> context.llvm.objects += irClass + ObjectStorageKind.PERMANENT -> { /* Do nothing, no need to free such an instance. */ } + } + val singleton = context.llvmDeclarations.forSingleton(irClass) + val instanceAddress = singleton.instanceStorage + val instanceShadowAddress = singleton.instanceShadowStorage + + if (storageKind == ObjectStorageKind.PERMANENT) { + return loadSlot(instanceAddress.getAddress(this), false) + } + val objectPtr = instanceAddress.getAddress(this) val bbInit = basicBlock("label_init", startLocationInfo, endLocationInfo) val bbExit = basicBlock("label_continue", startLocationInfo, endLocationInfo) val objectVal = loadSlot(objectPtr, false) @@ -954,7 +931,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, val ctor = codegen.llvmFunction(defaultConstructor) val (initFunction, args) = if (storageKind == ObjectStorageKind.SHARED && context.config.threadsAreAllowed) { - val shadowObjectPtr = codegen.getObjectInstanceShadowStorage(irClass) + val shadowObjectPtr = instanceShadowAddress!!.getAddress(this) context.llvm.initSharedInstanceFunction to listOf(objectPtr, shadowObjectPtr, typeInfo, ctor) } else { context.llvm.initInstanceFunction to listOf(objectPtr, typeInfo, ctor) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 340849ddada..a16c0a8cc26 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -157,7 +157,7 @@ internal interface ContextUtils : RuntimeAware { * It may be declared as external function prototype. */ val IrFunction.llvmFunction: LLVMValueRef - get() = llvmFunctionOrNull ?: error("$name in $file/${parent.fqNameForIrSerialization}") + get() = llvmFunctionOrNull ?: error("$name in $file/${parent.fqNameForIrSerialization}") val IrFunction.llvmFunctionOrNull: LLVMValueRef? get() { @@ -259,8 +259,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { throw IllegalArgumentException("function $name already exists") } - val externalFunction = LLVMGetNamedFunction(otherModule, name) ?: - throw Error("function $name not found") + val externalFunction = LLVMGetNamedFunction(otherModule, name) ?: throw Error("function $name not found") val functionType = getFunctionType(externalFunction) val function = LLVMAddFunction(llvmModule, name, functionType)!! @@ -315,10 +314,10 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { } internal fun externalFunction( - name: String, - type: LLVMTypeRef, - origin: CompiledKlibModuleOrigin, - independent: Boolean = false + name: String, + type: LLVMTypeRef, + origin: CompiledKlibModuleOrigin, + independent: Boolean = false ): LLVMValueRef { this.imports.add(origin, onlyBitcode = independent) @@ -451,6 +450,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val isInstanceOfClassFastFunction = importRtFunction("IsInstanceOfClassFast") val throwExceptionFunction = importRtFunction("ThrowException") val appendToInitalizersTail = importRtFunction("AppendToInitializersTail") + val addTLSRecord = importRtFunction("AddTLSRecord") + val clearTLSRecord = importRtFunction("ClearTLSRecord") + val lookupTLS = importRtFunction("LookupTLS") val initRuntimeIfNeeded = importRtFunction("Kotlin_initRuntimeIfNeeded") val mutationCheck = importRtFunction("MutationCheck") val freezeSubgraph = importRtFunction("FreezeSubgraph") @@ -486,6 +488,15 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { } } + var tlsCount = 0 + + val tlsKey by lazy { + val global = LLVMAddGlobal(llvmModule, kInt8Ptr, "__KonanTlsKey")!! + LLVMSetLinkage(global, LLVMLinkage.LLVMInternalLinkage) + LLVMSetInitializer(global, LLVMConstNull(kInt8Ptr)) + global + } + private val personalityFunctionName = when (target) { KonanTarget.IOS_ARM32 -> "__gxx_personality_sj0" KonanTarget.MINGW_X64 -> "__gxx_personality_seh0" @@ -535,8 +546,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val irStaticInitializers = mutableListOf() val otherStaticInitializers = mutableListOf() val fileInitializers = mutableListOf() - val objects = mutableSetOf() - val sharedObjects = mutableSetOf() + val objects = mutableSetOf() + val sharedObjects = mutableSetOf() private object lazyRtFunction { operator fun provideDelegate( @@ -548,6 +559,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { override fun getValue(thisRef: Llvm, property: KProperty<*>): LLVMValueRef = value } } + val llvmInt8 = int8Type val llvmInt16 = int16Type val llvmInt32 = int32Type diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 66644284394..51378f024fb 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -360,8 +360,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map 0) { + val memory = LLVMGetParam(initFunction, 1)!! + call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey, + Int32(context.llvm.tlsCount).llvm)) + } context.llvm.fileInitializers - .forEach { - if (it.initializer?.expression !is IrConst<*>?) { - if (it.storageKind != FieldStorageKind.THREAD_LOCAL) { - val initialization = evaluateExpression(it.initializer!!.expression) - val address = context.llvmDeclarations.forStaticField(it).storage - if (it.storageKind == FieldStorageKind.SHARED) + .forEach { irField -> + if (irField.initializer?.expression !is IrConst<*>?) { + if (irField.storageKind != FieldStorageKind.THREAD_LOCAL) { + val initialization = evaluateExpression(irField.initializer!!.expression) + val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress( + functionGenerationContext + ) + if (irField.storageKind == FieldStorageKind.SHARED) freeze(initialization, currentCodeContext.exceptionHandler) storeAny(initialization, address, false) } @@ -407,41 +417,50 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map 0) { + val memory = LLVMGetParam(initFunction, 1)!! + call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey, + Int32(context.llvm.tlsCount).llvm)) + } context.llvm.fileInitializers - .forEach { - if (it.initializer?.expression !is IrConst<*>?) { - if (it.storageKind == FieldStorageKind.THREAD_LOCAL) { - val initialization = evaluateExpression(it.initializer!!.expression) - val address = context.llvmDeclarations.forStaticField(it).storage - storeAny(initialization, address, false) - } + .forEach { irField -> + val expression = irField.initializer?.expression + if (irField.storageKind == FieldStorageKind.THREAD_LOCAL) { + val initialization = evaluateExpression(irField.initializer!!.expression) + val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress( + functionGenerationContext + ) + storeAny(initialization, address, false) } } ret(null) } appendingTo(bbLocalDeinit) { - context.llvm.fileInitializers.forEach { - // Only if a subject for memory management. - if (it.type.binaryTypeIsReference() && it.storageKind == FieldStorageKind.THREAD_LOCAL) { - val address = context.llvmDeclarations.forStaticField(it).storage - storeHeapRef(codegen.kNullObjHeaderPtr, address) - } + if (context.llvm.tlsCount > 0) { + val memory = LLVMGetParam(initFunction, 1)!! + call(context.llvm.clearTLSRecord, listOf(memory, context.llvm.tlsKey)) } - context.llvm.objects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) } ret(null) } appendingTo(bbGlobalDeinit) { context.llvm.fileInitializers // Only if a subject for memory management. - .forEach { - if (it.type.binaryTypeIsReference() && it.storageKind != FieldStorageKind.THREAD_LOCAL) { - val address = context.llvmDeclarations.forStaticField(it).storage + .forEach {irField -> + if (irField.type.binaryTypeIsReference() && irField.storageKind != FieldStorageKind.THREAD_LOCAL) { + val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress( + functionGenerationContext + ) storeHeapRef(codegen.kNullObjHeaderPtr, address) } } - context.llvm.sharedObjects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) } + context.llvm.sharedObjects.forEach { irClass -> + val address = context.llvmDeclarations.forSingleton(irClass).instanceStorage.getAddress( + functionGenerationContext + ) + storeHeapRef(codegen.kNullObjHeaderPtr, address) + } ret(null) } } @@ -751,10 +770,24 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map - if (initializer != null) - LLVMSetInitializer(globalProperty, evaluateExpression(initializer)) - else - LLVMSetInitializer(globalProperty, LLVMConstNull(type)) + val globalProperty = (globalPropertyAccess as? GlobalAddressAccess)?.getAddress(null) + if (globalProperty != null) { + LLVMSetInitializer(globalProperty, if (initializer != null) + evaluateExpression(initializer) else LLVMConstNull(type)) + // (Cannot do this before the global is initialized). + LLVMSetLinkage(globalProperty, LLVMLinkage.LLVMInternalLinkage) + } context.llvm.fileInitializers.add(declaration) - - // (Cannot do this before the global is initialized). - LLVMSetLinkage(globalProperty, LLVMLinkage.LLVMInternalLinkage) } } @@ -1515,7 +1548,9 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map() @@ -253,20 +254,51 @@ internal fun ContextUtils.importGlobal(name: String, type: LLVMTypeRef, origin: context.llvm.imports.add(origin) val found = LLVMGetNamedGlobal(context.llvmModule, name) - if (found != null) { + return if (found != null) { assert (getGlobalType(found) == type) assert (LLVMGetInitializer(found) == null) { "$name is already declared in the current module" } if (threadLocal) assert(LLVMGetThreadLocalMode(found) == context.llvm.tlsMode) - return found + found } else { - val result = LLVMAddGlobal(context.llvmModule, type, name)!! - if (threadLocal) - LLVMSetThreadLocalMode(result, context.llvm.tlsMode) - return result + addGlobal(name, type, false, threadLocal) } } +internal abstract class AddressAccess { + abstract fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef +} + +internal class GlobalAddressAccess(private val address: LLVMValueRef): AddressAccess() { + override fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef = address +} + +internal class TLSAddressAccess( + private val context: Context, private val index: Int): AddressAccess() { + + override fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef { + return generationContext!!.call(context.llvm.lookupTLS, + listOf(context.llvm.tlsKey, Int32(index).llvm)) + } +} + +internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, threadLocal: Boolean): AddressAccess { + if (threadLocal) { + return if (isObjectType(type)) { + val index = context.llvm.tlsCount++ + TLSAddressAccess(context, index) + } else { + GlobalAddressAccess(LLVMAddGlobal(context.llvmModule, type, name)!!.also { + LLVMSetThreadLocalMode(it, context.llvm.tlsMode) + LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) + }) + } + } + return GlobalAddressAccess(LLVMAddGlobal(context.llvmModule, type, name)!!.also { + LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage) + }) +} + internal fun functionType(returnType: LLVMTypeRef, isVarArg: Boolean = false, vararg paramTypes: LLVMTypeRef) = LLVMFunctionType( returnType, diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt index a666fd2cc92..de6ebe82476 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt @@ -29,6 +29,7 @@ class Runtime(bitcodeFile: String) { val objHeaderType = getStructType("ObjHeader") val objHeaderPtrType = pointerType(objHeaderType) + val objHeaderPtrPtrType = pointerType(objHeaderType) val arrayHeaderType = getStructType("ArrayHeader") val frameOverlayType = getStructType("FrameOverlay") diff --git a/backend.native/tests/codegen/object/fields2.kt b/backend.native/tests/codegen/object/fields2.kt index 38f55a00b71..defec5342a7 100644 --- a/backend.native/tests/codegen/object/fields2.kt +++ b/backend.native/tests/codegen/object/fields2.kt @@ -29,7 +29,7 @@ class TestClass { } } -@Test fun runTest() { +@Test fun runTest1() { global = 1 val test = TestClass() @@ -37,4 +37,19 @@ class TestClass { global = test.member test.member = global -} \ No newline at end of file +} + +@ThreadLocal +val xInt = 42 + +@ThreadLocal +val xString = "42" + +@ThreadLocal +val xAny = Any() + +@Test fun runTest2() { + assertEquals(42, xInt) + assertEquals("42", xString) + assertTrue(xAny is Any) +} diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 2cd01cbea0b..8861f72d3e7 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -96,6 +96,7 @@ typedef KStdUnorderedSet KRefSet; typedef KStdUnorderedMap KRefIntMap; typedef KStdDeque KRefDeque; typedef KStdDeque KRefListDeque; +typedef KStdUnorderedMap> KThreadLocalStorageMap; // A little hack that allows to enable -O2 optimizations // Prevents clang from replacing FrameOverlay struct @@ -412,6 +413,10 @@ struct MemoryState { ContainerHeaderSet* containers; #endif + KThreadLocalStorageMap* tlsMap; + KRef* tlsMapLastStart; + void* tlsMapLastKey; + #if USE_GC // Finalizer queue - linked list of containers scheduled for finalization. ContainerHeader* finalizerQueue; @@ -1734,6 +1739,7 @@ MemoryState* initMemory() { memoryState->allocSinceLastGcThreshold = kMaxGcAllocThreshold; memoryState->gcErgonomics = true; #endif + memoryState->tlsMap = konanConstructInstance(); memoryState->foreignRefManager = ForeignRefManager::create(); atomicAdd(&aliveMemoryStatesCount, 1); return memoryState; @@ -1751,6 +1757,8 @@ void deinitMemory(MemoryState* memoryState) { konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->roots); konanDestructInstance(memoryState->toRelease); + RuntimeAssert(memoryState->tlsMap->size() == 0, "Must be already cleared"); + konanDestructInstance(memoryState->tlsMap); RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty"); RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty"); @@ -3111,5 +3119,45 @@ void Konan_Platform_setMemoryLeakChecker(KBoolean value) { g_checkLeaks = value; } +void AddTLSRecord(MemoryState* memory, void** key, int size) { + auto* tlsMap = memory->tlsMap; + auto it = tlsMap->find(key); + if (it != tlsMap->end()) { + RuntimeAssert(it->second.second == size, "Size must be consistent"); + return; + } + KRef* start = reinterpret_cast(konanAllocMemory(size * sizeof(KRef))); + tlsMap->emplace(key, std::make_pair(start, size)); +} + +void ClearTLSRecord(MemoryState* memory, void** key) { + auto* tlsMap = memory->tlsMap; + auto it = tlsMap->find(key); + if (it != tlsMap->end()) { + KRef* start = it->second.first; + int count = it->second.second; + for (int i = 0; i < count; i++) { + UpdateHeapRef(start + i, nullptr); + } + konanFreeMemory(start); + tlsMap->erase(it); + } +} + +KRef* LookupTLS(void** key, int index) { + auto* state = memoryState; + auto* tlsMap = state->tlsMap; + // In many cases there is only one module, so this one element cache. + if (state->tlsMapLastKey == key) { + return state->tlsMapLastStart + index; + } + auto it = tlsMap->find(key); + RuntimeAssert(it != tlsMap->end(), "Must be there"); + RuntimeAssert(index < it->second.second, "Out of bound in TLS access"); + KRef* start = it->second.first; + state->tlsMapLastKey = key; + state->tlsMapLastStart = start; + return start + index; +} } // extern "C" diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index df9272385e5..a83b024b17e 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -548,6 +548,13 @@ void MutationCheck(ObjHeader* obj); void FreezeSubgraph(ObjHeader* obj); // Ensure this object shall block freezing. void EnsureNeverFrozen(ObjHeader* obj); +// Add TLS object storage, called by the generated code. +void AddTLSRecord(MemoryState* memory, void** key, int size) RUNTIME_NOTHROW; +// Clear TLS object storage, called by the generated code. +void ClearTLSRecord(MemoryState* memory, void** key) RUNTIME_NOTHROW; +// Lookup element in TLS object storage. +ObjHeader** LookupTLS(void** key, int index) RUNTIME_NOTHROW; + #ifdef __cplusplus } #endif diff --git a/runtime/src/main/cpp/Runtime.cpp b/runtime/src/main/cpp/Runtime.cpp index 0431881db36..801f16fe7ab 100644 --- a/runtime/src/main/cpp/Runtime.cpp +++ b/runtime/src/main/cpp/Runtime.cpp @@ -30,7 +30,7 @@ struct RuntimeState { volatile int executionStatus; }; -typedef void (*Initializer)(int initialize); +typedef void (*Initializer)(int initialize, MemoryState* memory); struct InitNode { Initializer init; InitNode* next; @@ -66,11 +66,11 @@ bool updateStatusIf(RuntimeState* state, int oldStatus, int newStatus) { #endif } -void InitOrDeinitGlobalVariables(int initialize) { - InitNode *currNode = initHeadNode; - while (currNode != nullptr) { - currNode->init(initialize); - currNode = currNode->next; +void InitOrDeinitGlobalVariables(int initialize, MemoryState* memory) { + InitNode* currentNode = initHeadNode; + while (currentNode != nullptr) { + currentNode->init(initialize, memory); + currentNode = currentNode->next; } } @@ -98,23 +98,21 @@ RuntimeState* initRuntime() { if (firstRuntime) { isMainThread = 1; konan::consoleInit(); - #if KONAN_OBJC_INTEROP Kotlin_ObjCExport_initialize(); #endif - - InitOrDeinitGlobalVariables(INIT_GLOBALS); + InitOrDeinitGlobalVariables(INIT_GLOBALS, result->memoryState); } - InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS); + InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS, result->memoryState); return result; } void deinitRuntime(RuntimeState* state) { ResumeMemory(state->memoryState); bool lastRuntime = atomicAdd(&aliveRuntimesCount, -1) == 0; - InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS); + InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS, state->memoryState); if (lastRuntime) - InitOrDeinitGlobalVariables(DEINIT_GLOBALS); + InitOrDeinitGlobalVariables(DEINIT_GLOBALS, state->memoryState); WorkerDeinit(state->worker); DeinitMemory(state->memoryState); konanDestructInstance(state); @@ -266,7 +264,8 @@ KBoolean Konan_Platform_isDebugBinary() { } void Kotlin_zeroOutTLSGlobals() { - InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS); + if (runtimeState != nullptr && runtimeState->memoryState != nullptr) + InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS, runtimeState->memoryState); } } // extern "C"