Rework TLS access. (#3678)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt

@@ -21,6 +21,8 @@ import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.ir.declarations.IrEnumEntry
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.util.SymbolTable
+import org.jetbrains.kotlin.ir.util.isEnumClass
+import org.jetbrains.kotlin.ir.util.isEnumEntry
 import org.jetbrains.kotlin.ir.util.referenceFunction
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.name.isChildOf

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -348,9 +348,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
 
     val coverage = CoverageManager(this)
 
-    // Cache used for source offset->(line,column) mapping.
-    val fileEntryCache = mutableMapOf<String, SourceManager.FileEntry>()
-
     protected fun separator(title: String) {
         println("\n\n--- ${title} ----------------------\n")
     }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt

@@ -129,22 +129,13 @@ internal val IrClass.writableTypeInfoSymbolName: String
         return "ktypew:" + this.fqNameForIrSerialization.toString()
     }
 
-internal val IrClass.objectInstanceFieldSymbolName: String
+internal val IrClass.objectInstanceGetterSymbolName: String
     get() {
         assert (this.isExported())
         assert (this.kind.isSingleton)
         assert (!this.isUnit())
 
-        return "kobjref:$fqNameForIrSerialization"
-    }
-
-internal val IrClass.objectInstanceShadowFieldSymbolName: String
-    get() {
-        assert (this.isExported())
-        assert (this.kind.isSingleton)
-        assert (!this.isUnit())
-
-        return "kshadowobjref:$fqNameForIrSerialization"
+        return "kobjget:$fqNameForIrSerialization"
     }
 
 val IrFunction.functionName get() = with(KonanMangler) { functionName }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -64,46 +64,6 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
     fun functionEntryPointAddress(function: IrFunction) = function.entryPointAddress.llvm
     fun functionHash(function: IrFunction): LLVMValueRef = function.functionName.localHash.llvm
 
-    fun getObjectInstanceStorage(irClass: IrClass, kind: ObjectStorageKind): LLVMValueRef {
-        assert (!irClass.isUnit())
-        val llvmGlobal = if (!isExternal(irClass)) {
-            context.llvmDeclarations.forSingleton(irClass).instanceFieldRef
-        } else {
-            val llvmType = getLLVMType(irClass.defaultType)
-            importGlobal(
-                    irClass.objectInstanceFieldSymbolName,
-                    llvmType,
-                    origin = irClass.llvmSymbolOrigin,
-                    threadLocal = kind == ObjectStorageKind.THREAD_LOCAL
-            )
-        }
-        when (kind) {
-            ObjectStorageKind.SHARED -> context.llvm.sharedObjects += llvmGlobal
-            ObjectStorageKind.THREAD_LOCAL -> context.llvm.objects += llvmGlobal
-            ObjectStorageKind.PERMANENT -> { /* Do nothing, no need to free such an instance. */ }
-        }
-
-        return llvmGlobal
-    }
-
-    fun getObjectInstanceShadowStorage(irClass: IrClass): LLVMValueRef {
-        assert (!irClass.isUnit())
-        assert (irClass.storageKind(context) == ObjectStorageKind.SHARED)
-        val llvmGlobal = if (!isExternal(irClass)) {
-            context.llvmDeclarations.forSingleton(irClass).instanceShadowFieldRef!!
-        } else {
-            val llvmType = getLLVMType(irClass.defaultType)
-            importGlobal(
-                    irClass.objectInstanceShadowFieldSymbolName,
-                    llvmType,
-                    origin = irClass.llvmSymbolOrigin,
-                    threadLocal = true
-            )
-        }
-        context.llvm.objects += llvmGlobal
-        return llvmGlobal
-    }
-
     fun typeInfoForAllocation(constructedClass: IrClass): LLVMValueRef {
         assert(!constructedClass.isObjCClass())
         return typeInfoValue(constructedClass)
@@ -911,11 +871,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         return null
     }
 
-    fun getObjectValue(
-            irClass: IrClass,
-            exceptionHandler: ExceptionHandler,
-            startLocationInfo: LocationInfo?,
-            endLocationInfo: LocationInfo? = null
+    fun getObjectValue(irClass: IrClass, exceptionHandler: ExceptionHandler,
+            startLocationInfo: LocationInfo?, endLocationInfo: LocationInfo? = null
     ): LLVMValueRef {
         // TODO: could be processed the same way as other stateless objects.
         if (irClass.isUnit()) {
@@ -926,7 +883,6 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
             val parent = irClass.parent as IrClass
             if (parent.isObjCClass()) {
                 // TODO: cache it too.
-
                 return call(
                         codegen.llvmFunction(context.ir.symbols.interopInterpretObjCPointer.owner),
                         listOf(getObjCClass(parent, exceptionHandler)),
@@ -935,12 +891,33 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
                 )
             }
         }
-        val storageKind = irClass.storageKind(context)
-        if (storageKind == ObjectStorageKind.PERMANENT) {
-            return loadSlot(codegen.getObjectInstanceStorage(irClass, storageKind), false)
+
+        // If object is imported - access it via getter function.
+        if (isExternal(irClass)) {
+            val valueGetterName = irClass.objectInstanceGetterSymbolName
+            val valueGetterFunction = LLVMGetNamedFunction(context.llvmModule, valueGetterName) ?:
+                LLVMAddFunction(context.llvmModule, valueGetterName,
+                        functionType(kObjHeaderPtr, false, kObjHeaderPtrPtr))
+            return call(valueGetterFunction!!,
+                        listOf(),
+                        resultLifetime = Lifetime.GLOBAL,
+                        exceptionHandler = exceptionHandler)
         }
 
-        val objectPtr = codegen.getObjectInstanceStorage(irClass, storageKind)
+        val storageKind = irClass.storageKind(context)
+        when (storageKind) {
+            ObjectStorageKind.SHARED -> context.llvm.sharedObjects += irClass
+            ObjectStorageKind.THREAD_LOCAL -> context.llvm.objects += irClass
+            ObjectStorageKind.PERMANENT -> { /* Do nothing, no need to free such an instance. */ }
+        }
+        val singleton = context.llvmDeclarations.forSingleton(irClass)
+        val instanceAddress = singleton.instanceStorage
+        val instanceShadowAddress = singleton.instanceShadowStorage
+
+        if (storageKind == ObjectStorageKind.PERMANENT) {
+            return loadSlot(instanceAddress.getAddress(this), false)
+        }
+        val objectPtr = instanceAddress.getAddress(this)
         val bbInit = basicBlock("label_init", startLocationInfo, endLocationInfo)
         val bbExit = basicBlock("label_continue", startLocationInfo, endLocationInfo)
         val objectVal = loadSlot(objectPtr, false)
@@ -954,7 +931,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         val ctor = codegen.llvmFunction(defaultConstructor)
         val (initFunction, args) =
                 if (storageKind == ObjectStorageKind.SHARED && context.config.threadsAreAllowed) {
-                    val shadowObjectPtr = codegen.getObjectInstanceShadowStorage(irClass)
+                    val shadowObjectPtr = instanceShadowAddress!!.getAddress(this)
                     context.llvm.initSharedInstanceFunction to listOf(objectPtr, shadowObjectPtr, typeInfo, ctor)
                 } else {
                     context.llvm.initInstanceFunction to listOf(objectPtr, typeInfo, ctor)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -157,7 +157,7 @@ internal interface ContextUtils : RuntimeAware {
      * It may be declared as external function prototype.
      */
     val IrFunction.llvmFunction: LLVMValueRef
-    get() = llvmFunctionOrNull ?: error("$name in $file/${parent.fqNameForIrSerialization}")
+        get() = llvmFunctionOrNull ?: error("$name in $file/${parent.fqNameForIrSerialization}")
 
     val IrFunction.llvmFunctionOrNull: LLVMValueRef?
         get() {
@@ -259,8 +259,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
             throw IllegalArgumentException("function $name already exists")
         }
 
-        val externalFunction = LLVMGetNamedFunction(otherModule, name) ?:
-            throw Error("function $name not found")
+        val externalFunction = LLVMGetNamedFunction(otherModule, name) ?: throw Error("function $name not found")
 
         val functionType = getFunctionType(externalFunction)
         val function = LLVMAddFunction(llvmModule, name, functionType)!!
@@ -315,10 +314,10 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     }
 
     internal fun externalFunction(
-        name: String,
-        type: LLVMTypeRef,
-        origin: CompiledKlibModuleOrigin,
-        independent: Boolean = false
+            name: String,
+            type: LLVMTypeRef,
+            origin: CompiledKlibModuleOrigin,
+            independent: Boolean = false
     ): LLVMValueRef {
         this.imports.add(origin, onlyBitcode = independent)
 
@@ -451,6 +450,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val isInstanceOfClassFastFunction = importRtFunction("IsInstanceOfClassFast")
     val throwExceptionFunction = importRtFunction("ThrowException")
     val appendToInitalizersTail = importRtFunction("AppendToInitializersTail")
+    val addTLSRecord = importRtFunction("AddTLSRecord")
+    val clearTLSRecord = importRtFunction("ClearTLSRecord")
+    val lookupTLS = importRtFunction("LookupTLS")
     val initRuntimeIfNeeded = importRtFunction("Kotlin_initRuntimeIfNeeded")
     val mutationCheck = importRtFunction("MutationCheck")
     val freezeSubgraph = importRtFunction("FreezeSubgraph")
@@ -486,6 +488,15 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
         }
     }
 
+    var tlsCount = 0
+
+    val tlsKey by lazy {
+        val global = LLVMAddGlobal(llvmModule, kInt8Ptr, "__KonanTlsKey")!!
+        LLVMSetLinkage(global, LLVMLinkage.LLVMInternalLinkage)
+        LLVMSetInitializer(global, LLVMConstNull(kInt8Ptr))
+        global
+    }
+
     private val personalityFunctionName = when (target) {
         KonanTarget.IOS_ARM32 -> "__gxx_personality_sj0"
         KonanTarget.MINGW_X64 -> "__gxx_personality_seh0"
@@ -535,8 +546,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val irStaticInitializers = mutableListOf<IrStaticInitializer>()
     val otherStaticInitializers = mutableListOf<LLVMValueRef>()
     val fileInitializers = mutableListOf<IrField>()
-    val objects = mutableSetOf<LLVMValueRef>()
-    val sharedObjects = mutableSetOf<LLVMValueRef>()
+    val objects = mutableSetOf<IrClass>()
+    val sharedObjects = mutableSetOf<IrClass>()
 
     private object lazyRtFunction {
         operator fun provideDelegate(
@@ -548,6 +559,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
             override fun getValue(thisRef: Llvm, property: KProperty<*>): LLVMValueRef = value
         }
     }
+
     val llvmInt8 = int8Type
     val llvmInt16 = int16Type
     val llvmInt32 = int32Type

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -360,8 +360,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     //-------------------------------------------------------------------------//
 
     val kVoidFuncType = functionType(voidType)
-    val kInitFuncType = functionType(voidType, false, int32Type)
     val kNodeInitType = LLVMGetTypeByName(context.llvmModule, "struct.InitNode")!!
+    val kMemoryStateType = LLVMGetTypeByName(context.llvmModule, "struct.MemoryState")!!
+    val kInitFuncType = functionType(voidType, false, int32Type, pointerType(kMemoryStateType))
+
     //-------------------------------------------------------------------------//
 
     val INIT_GLOBALS = 0
@@ -391,13 +393,21 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
                 // Globals initalizers may contain accesses to objects, so visit them first.
                 appendingTo(bbInit) {
+                    // Bit clumsy, global init may need access to TLS, thus it has to be ready to that point.
+                    if (context.llvm.tlsCount > 0) {
+                        val memory = LLVMGetParam(initFunction, 1)!!
+                        call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey,
+                                Int32(context.llvm.tlsCount).llvm))
+                    }
                     context.llvm.fileInitializers
-                            .forEach {
-                                if (it.initializer?.expression !is IrConst<*>?) {
-                                    if (it.storageKind != FieldStorageKind.THREAD_LOCAL) {
-                                        val initialization = evaluateExpression(it.initializer!!.expression)
-                                        val address = context.llvmDeclarations.forStaticField(it).storage
-                                        if (it.storageKind == FieldStorageKind.SHARED)
+                            .forEach { irField ->
+                                if (irField.initializer?.expression !is IrConst<*>?) {
+                                    if (irField.storageKind != FieldStorageKind.THREAD_LOCAL) {
+                                        val initialization = evaluateExpression(irField.initializer!!.expression)
+                                        val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress(
+                                                functionGenerationContext
+                                        )
+                                        if (irField.storageKind == FieldStorageKind.SHARED)
                                             freeze(initialization, currentCodeContext.exceptionHandler)
                                         storeAny(initialization, address, false)
                                     }
@@ -407,41 +417,50 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
                 }
 
                 appendingTo(bbLocalInit) {
+                    if (context.llvm.tlsCount > 0) {
+                        val memory = LLVMGetParam(initFunction, 1)!!
+                        call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey,
+                                Int32(context.llvm.tlsCount).llvm))
+                    }
                     context.llvm.fileInitializers
-                            .forEach {
-                                if (it.initializer?.expression !is IrConst<*>?) {
-                                   if (it.storageKind == FieldStorageKind.THREAD_LOCAL) {
-                                       val initialization = evaluateExpression(it.initializer!!.expression)
-                                       val address = context.llvmDeclarations.forStaticField(it).storage
-                                       storeAny(initialization, address, false)
-                                    }
+                            .forEach { irField ->
+                                val expression = irField.initializer?.expression
+                                if (irField.storageKind == FieldStorageKind.THREAD_LOCAL) {
+                                    val initialization = evaluateExpression(irField.initializer!!.expression)
+                                    val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress(
+                                            functionGenerationContext
+                                    )
+                                    storeAny(initialization, address, false)
                                 }
                             }
                     ret(null)
                 }
 
                 appendingTo(bbLocalDeinit) {
-                    context.llvm.fileInitializers.forEach {
-                        // Only if a subject for memory management.
-                        if (it.type.binaryTypeIsReference() && it.storageKind == FieldStorageKind.THREAD_LOCAL) {
-                            val address = context.llvmDeclarations.forStaticField(it).storage
-                            storeHeapRef(codegen.kNullObjHeaderPtr, address)
-                        }
+                    if (context.llvm.tlsCount > 0) {
+                        val memory = LLVMGetParam(initFunction, 1)!!
+                        call(context.llvm.clearTLSRecord, listOf(memory, context.llvm.tlsKey))
                     }
-                    context.llvm.objects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) }
                     ret(null)
                 }
 
                 appendingTo(bbGlobalDeinit) {
                     context.llvm.fileInitializers
                             // Only if a subject for memory management.
-                            .forEach {
-                                if (it.type.binaryTypeIsReference() && it.storageKind != FieldStorageKind.THREAD_LOCAL) {
-                                    val address = context.llvmDeclarations.forStaticField(it).storage
+                            .forEach {irField ->
+                                if (irField.type.binaryTypeIsReference() && irField.storageKind != FieldStorageKind.THREAD_LOCAL) {
+                                    val address = context.llvmDeclarations.forStaticField(irField).storageAddressAccess.getAddress(
+                                            functionGenerationContext
+                                    )
                                     storeHeapRef(codegen.kNullObjHeaderPtr, address)
                                 }
                             }
-                    context.llvm.sharedObjects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) }
+                    context.llvm.sharedObjects.forEach { irClass ->
+                        val address = context.llvmDeclarations.forSingleton(irClass).instanceStorage.getAddress(
+                                functionGenerationContext
+                        )
+                        storeHeapRef(codegen.kNullObjHeaderPtr, address)
+                    }
                     ret(null)
                 }
             }
@@ -751,10 +770,24 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         }
 
         if (declaration.kind.isSingleton && !declaration.isUnit()) {
-            val value = context.llvmDeclarations.forSingleton(declaration).instanceFieldRef
-            LLVMSetInitializer(value, if (declaration.storageKind(context) == ObjectStorageKind.PERMANENT)
-                context.llvm.staticData.createConstKotlinObject(declaration,
-                        *computeFields(declaration)).llvm else codegen.kNullObjHeaderPtr)
+            val singleton = context.llvmDeclarations.forSingleton(declaration)
+            val access = singleton.instanceStorage
+            if (access is GlobalAddressAccess) {
+                LLVMSetInitializer(access.getAddress(null), if (declaration.storageKind(context) == ObjectStorageKind.PERMANENT)
+                    context.llvm.staticData.createConstKotlinObject(declaration,
+                            *computeFields(declaration)).llvm else codegen.kNullObjHeaderPtr)
+            }
+            // If can be exported and can be instantiated.
+            if (declaration.isExported() &&
+                    declaration.constructors.singleOrNull() { it.valueParameters.size == 0 } != null) {
+                val valueGetterName = declaration.objectInstanceGetterSymbolName
+                generateFunction(codegen,
+                        LLVMFunctionType(codegen.kObjHeaderPtr, cValuesOf(codegen.kObjHeaderPtrPtr), 1, 0)!!,
+                        valueGetterName) {
+                    val value = getObjectValue(declaration, ExceptionHandler.Caller, null, null)
+                    ret(value)
+                }
+            }
         }
     }
 
@@ -779,16 +812,16 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         debugFieldDeclaration(declaration)
         if (context.needGlobalInit(declaration)) {
             val type = codegen.getLLVMType(declaration.type)
-            val globalProperty = context.llvmDeclarations.forStaticField(declaration).storage
+            val globalPropertyAccess = context.llvmDeclarations.forStaticField(declaration).storageAddressAccess
             val initializer = declaration.initializer?.expression as? IrConst<*>
-            if (initializer != null)
-                LLVMSetInitializer(globalProperty, evaluateExpression(initializer))
-            else
-                LLVMSetInitializer(globalProperty, LLVMConstNull(type))
+            val globalProperty = (globalPropertyAccess as? GlobalAddressAccess)?.getAddress(null)
+            if (globalProperty != null) {
+                LLVMSetInitializer(globalProperty, if (initializer != null)
+                    evaluateExpression(initializer) else LLVMConstNull(type))
+                // (Cannot do this before the global is initialized).
+                LLVMSetLinkage(globalProperty, LLVMLinkage.LLVMInternalLinkage)
+            }
             context.llvm.fileInitializers.add(declaration)
-
-            // (Cannot do this before the global is initialized).
-            LLVMSetLinkage(globalProperty, LLVMLinkage.LLVMInternalLinkage)
         }
     }
 
@@ -1515,7 +1548,9 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
                 if (context.config.threadsAreAllowed && value.symbol.owner.isMainOnlyNonPrimitive) {
                     functionGenerationContext.checkMainThread(currentCodeContext.exceptionHandler)
                 }
-                val ptr = context.llvmDeclarations.forStaticField(value.symbol.owner).storage
+                val ptr = context.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress(
+                        functionGenerationContext
+                )
                 functionGenerationContext.loadSlot(ptr, !value.symbol.owner.isFinal)
             }
         }.also {
@@ -1548,12 +1583,14 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner), false)
         } else {
             assert(value.receiver == null)
-            val globalValue = context.llvmDeclarations.forStaticField(value.symbol.owner).storage
+            val globalAddress = context.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress(
+                    functionGenerationContext
+            )
             if (context.config.threadsAreAllowed && value.symbol.owner.isMainOnlyNonPrimitive)
                 functionGenerationContext.checkMainThread(currentCodeContext.exceptionHandler)
             if (value.symbol.owner.storageKind == FieldStorageKind.SHARED)
                 functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler)
-            functionGenerationContext.storeAny(valueToAssign, globalValue, false)
+            functionGenerationContext.storeAny(valueToAssign, globalAddress, false)
         }
         if (store != null && value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true) {
             LLVMSetAlignment(store, 8)
@@ -2420,6 +2457,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             val bbInited = basicBlock("inited", null)
             val bbNeedInit = basicBlock("need_init", null)
 
+
             val value = LLVMBuildLoad(builder, initGuard, "")!!
             condBr(icmpEq(value, kImmZero), bbNeedInit, bbInited)
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt

@@ -67,7 +67,7 @@ internal class ClassLlvmDeclarations(
         val singletonDeclarations: SingletonLlvmDeclarations?,
         val objCDeclarations: KotlinObjCClassLlvmDeclarations?)
 
-internal class SingletonLlvmDeclarations(val instanceFieldRef: LLVMValueRef, val instanceShadowFieldRef: LLVMValueRef?)
+internal class SingletonLlvmDeclarations(val instanceStorage: AddressAccess, val instanceShadowStorage: AddressAccess?)
 
 internal class KotlinObjCClassLlvmDeclarations(
         val classPointerGlobal: StaticData.Global,
@@ -79,7 +79,7 @@ internal class FunctionLlvmDeclarations(val llvmFunction: LLVMValueRef)
 
 internal class FieldLlvmDeclarations(val index: Int, val classBodyType: LLVMTypeRef)
 
-internal class StaticFieldLlvmDeclarations(val storage: LLVMValueRef)
+internal class StaticFieldLlvmDeclarations(val storageAddressAccess: AddressAccess)
 
 internal class UniqueLlvmDeclarations(val pointer: ConstPointer)
 
@@ -262,35 +262,28 @@ private class DeclarationsGeneratorVisitor(override val context: Context) :
     }
 
     private fun createSingletonDeclarations(irClass: IrClass): SingletonLlvmDeclarations? {
-
         if (irClass.isUnit()) {
             return null
         }
 
         val isExported = irClass.isExported()
-        val symbolName = if (isExported) {
-            irClass.objectInstanceFieldSymbolName
+        val valueGetterName = if (isExported) {
+            irClass.objectInstanceGetterSymbolName
         } else {
-            "kobjref:" + qualifyInternalName(irClass)
+            null
         }
-        val threadLocal = irClass.storageKind(context) == ObjectStorageKind.THREAD_LOCAL
-        val instanceFieldRef = addGlobal(
-                symbolName, getLLVMType(irClass.defaultType), isExported = isExported, threadLocal = threadLocal)
 
-        val instanceShadowFieldRef =
-                if (threadLocal) null
-                else {
-                    val shadowSymbolName = if (isExported) {
-                        irClass.objectInstanceShadowFieldSymbolName
-                    } else {
-                        "kshadowobjref:" + qualifyInternalName(irClass)
-                    }
-                    addGlobal(shadowSymbolName, getLLVMType(irClass.defaultType), isExported = isExported, threadLocal = true)
-                }
+        val storageKind = irClass.storageKind(context)
+        val threadLocal = storageKind == ObjectStorageKind.THREAD_LOCAL
+        val symbolName = "kobjref:" + qualifyInternalName(irClass)
+        val instanceAddress = addKotlinGlobal(symbolName, getLLVMType(irClass.defaultType), threadLocal = threadLocal)
 
-        instanceShadowFieldRef?.let { LLVMSetInitializer(it, kNullObjHeaderPtr) }
+        val instanceShadowAddress = if (threadLocal || storageKind == ObjectStorageKind.PERMANENT) null else {
+            val shadowSymbolName = "kshadowobjref:" + qualifyInternalName(irClass)
+            addKotlinGlobal(shadowSymbolName, getLLVMType(irClass.defaultType), threadLocal = true)
+        }
 
-        return SingletonLlvmDeclarations(instanceFieldRef, instanceShadowFieldRef)
+        return SingletonLlvmDeclarations(instanceAddress, instanceShadowAddress)
     }
 
     private fun createKotlinObjCClassDeclarations(irClass: IrClass): KotlinObjCClassLlvmDeclarations {
@@ -325,9 +318,8 @@ private class DeclarationsGeneratorVisitor(override val context: Context) :
         } else {
             // Fields are module-private, so we use internal name:
             val name = "kvar:" + qualifyInternalName(declaration)
-            val storage = addGlobal(
-                    name, getLLVMType(declaration.type), isExported = false,
-                    threadLocal = declaration.storageKind == FieldStorageKind.THREAD_LOCAL)
+            val storage = addKotlinGlobal(
+                    name, getLLVMType(declaration.type), threadLocal = declaration.storageKind == FieldStorageKind.THREAD_LOCAL)
 
             this.staticFields[declaration] = StaticFieldLlvmDeclarations(storage)
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.backend.konan.llvm
 
 import kotlinx.cinterop.*
 import llvm.*
+import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.descriptors.konan.CompiledKlibModuleOrigin
 
 private val llvmContextHolder = ThreadLocal<LLVMContextRef>()
@@ -253,20 +254,51 @@ internal fun ContextUtils.importGlobal(name: String, type: LLVMTypeRef, origin:
     context.llvm.imports.add(origin)
 
     val found = LLVMGetNamedGlobal(context.llvmModule, name)
-    if (found != null) {
+    return if (found != null) {
         assert (getGlobalType(found) == type)
         assert (LLVMGetInitializer(found) == null) { "$name is already declared in the current module" }
         if (threadLocal)
             assert(LLVMGetThreadLocalMode(found) == context.llvm.tlsMode)
-        return found
+        found
     } else {
-        val result = LLVMAddGlobal(context.llvmModule, type, name)!!
-        if (threadLocal)
-            LLVMSetThreadLocalMode(result, context.llvm.tlsMode)
-        return result
+        addGlobal(name, type, false, threadLocal)
     }
 }
 
+internal abstract class AddressAccess {
+    abstract fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef
+}
+
+internal class GlobalAddressAccess(private val address: LLVMValueRef): AddressAccess() {
+    override fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef = address
+}
+
+internal class TLSAddressAccess(
+        private val context: Context, private val index: Int): AddressAccess() {
+
+    override fun getAddress(generationContext: FunctionGenerationContext?): LLVMValueRef {
+        return generationContext!!.call(context.llvm.lookupTLS,
+                listOf(context.llvm.tlsKey, Int32(index).llvm))
+    }
+}
+
+internal fun ContextUtils.addKotlinGlobal(name: String, type: LLVMTypeRef, threadLocal: Boolean): AddressAccess {
+    if (threadLocal) {
+        return if (isObjectType(type)) {
+            val index = context.llvm.tlsCount++
+            TLSAddressAccess(context, index)
+        } else {
+            GlobalAddressAccess(LLVMAddGlobal(context.llvmModule, type, name)!!.also {
+                LLVMSetThreadLocalMode(it, context.llvm.tlsMode)
+                LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage)
+            })
+        }
+    }
+    return GlobalAddressAccess(LLVMAddGlobal(context.llvmModule, type, name)!!.also {
+        LLVMSetLinkage(it, LLVMLinkage.LLVMInternalLinkage)
+    })
+}
+
 internal fun functionType(returnType: LLVMTypeRef, isVarArg: Boolean = false, vararg paramTypes: LLVMTypeRef) =
         LLVMFunctionType(
                 returnType,

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt

@@ -29,6 +29,7 @@ class Runtime(bitcodeFile: String) {
 
     val objHeaderType = getStructType("ObjHeader")
     val objHeaderPtrType = pointerType(objHeaderType)
+    val objHeaderPtrPtrType = pointerType(objHeaderType)
     val arrayHeaderType = getStructType("ArrayHeader")
 
     val frameOverlayType = getStructType("FrameOverlay")

backend.native/tests/codegen/object/fields2.kt

@@ -29,7 +29,7 @@ class TestClass {
         }
 }
 
-@Test fun runTest() {
+@Test fun runTest1() {
     global = 1
 
     val test = TestClass()
@@ -37,4 +37,19 @@ class TestClass {
 
     global = test.member
     test.member = global
-}
+}
+
+@ThreadLocal
+val xInt = 42
+
+@ThreadLocal
+val xString = "42"
+
+@ThreadLocal
+val xAny = Any()
+
+@Test fun runTest2() {
+    assertEquals(42, xInt)
+    assertEquals("42", xString)
+    assertTrue(xAny is Any)
+}

runtime/src/main/cpp/Memory.cpp

@@ -96,6 +96,7 @@ typedef KStdUnorderedSet<KRef> KRefSet;
 typedef KStdUnorderedMap<KRef, KInt> KRefIntMap;
 typedef KStdDeque<KRef> KRefDeque;
 typedef KStdDeque<KRefList> KRefListDeque;
+typedef KStdUnorderedMap<void**, std::pair<KRef*,int>> KThreadLocalStorageMap;
 
 // A little hack that allows to enable -O2 optimizations
 // Prevents clang from replacing FrameOverlay struct
@@ -412,6 +413,10 @@ struct MemoryState {
   ContainerHeaderSet* containers;
 #endif
 
+  KThreadLocalStorageMap* tlsMap;
+  KRef* tlsMapLastStart;
+  void* tlsMapLastKey;
+
 #if USE_GC
   // Finalizer queue - linked list of containers scheduled for finalization.
   ContainerHeader* finalizerQueue;
@@ -1734,6 +1739,7 @@ MemoryState* initMemory() {
   memoryState->allocSinceLastGcThreshold = kMaxGcAllocThreshold;
   memoryState->gcErgonomics = true;
 #endif
+  memoryState->tlsMap = konanConstructInstance<KThreadLocalStorageMap>();
   memoryState->foreignRefManager = ForeignRefManager::create();
   atomicAdd(&aliveMemoryStatesCount, 1);
   return memoryState;
@@ -1751,6 +1757,8 @@ void deinitMemory(MemoryState* memoryState) {
   konanDestructInstance(memoryState->toFree);
   konanDestructInstance(memoryState->roots);
   konanDestructInstance(memoryState->toRelease);
+  RuntimeAssert(memoryState->tlsMap->size() == 0, "Must be already cleared");
+  konanDestructInstance(memoryState->tlsMap);
   RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty");
   RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty");
 
@@ -3111,5 +3119,45 @@ void Konan_Platform_setMemoryLeakChecker(KBoolean value) {
   g_checkLeaks = value;
 }
 
+void AddTLSRecord(MemoryState* memory, void** key, int size) {
+  auto* tlsMap = memory->tlsMap;
+  auto it = tlsMap->find(key);
+  if (it != tlsMap->end()) {
+    RuntimeAssert(it->second.second == size, "Size must be consistent");
+    return;
+  }
+  KRef* start = reinterpret_cast<KRef*>(konanAllocMemory(size * sizeof(KRef)));
+  tlsMap->emplace(key, std::make_pair(start, size));
+}
+
+void ClearTLSRecord(MemoryState* memory, void** key) {
+  auto* tlsMap = memory->tlsMap;
+  auto it = tlsMap->find(key);
+  if (it != tlsMap->end()) {
+    KRef* start = it->second.first;
+    int count = it->second.second;
+    for (int i = 0; i < count; i++) {
+      UpdateHeapRef(start + i, nullptr);
+    }
+    konanFreeMemory(start);
+    tlsMap->erase(it);
+  }
+}
+
+KRef* LookupTLS(void** key, int index) {
+  auto* state = memoryState;
+  auto* tlsMap = state->tlsMap;
+  // In many cases there is only one module, so this one element cache.
+  if (state->tlsMapLastKey == key) {
+    return state->tlsMapLastStart + index;
+  }
+  auto it = tlsMap->find(key);
+  RuntimeAssert(it != tlsMap->end(), "Must be there");
+  RuntimeAssert(index < it->second.second, "Out of bound in TLS access");
+  KRef* start = it->second.first;
+  state->tlsMapLastKey = key;
+  state->tlsMapLastStart = start;
+  return start + index;
+}
 
 } // extern "C"

runtime/src/main/cpp/Memory.h

@@ -548,6 +548,13 @@ void MutationCheck(ObjHeader* obj);
 void FreezeSubgraph(ObjHeader* obj);
 // Ensure this object shall block freezing.
 void EnsureNeverFrozen(ObjHeader* obj);
+// Add TLS object storage, called by the generated code.
+void AddTLSRecord(MemoryState* memory, void** key, int size) RUNTIME_NOTHROW;
+// Clear TLS object storage, called by the generated code.
+void ClearTLSRecord(MemoryState* memory, void** key) RUNTIME_NOTHROW;
+// Lookup element in TLS object storage.
+ObjHeader** LookupTLS(void** key, int index) RUNTIME_NOTHROW;
+
 #ifdef __cplusplus
 }
 #endif

runtime/src/main/cpp/Runtime.cpp

@@ -30,7 +30,7 @@ struct RuntimeState {
   volatile int executionStatus;
 };
 
-typedef void (*Initializer)(int initialize);
+typedef void (*Initializer)(int initialize, MemoryState* memory);
 struct InitNode {
   Initializer init;
   InitNode* next;
@@ -66,11 +66,11 @@ bool updateStatusIf(RuntimeState* state, int oldStatus, int newStatus) {
 #endif
 }
 
-void InitOrDeinitGlobalVariables(int initialize) {
-  InitNode *currNode = initHeadNode;
-  while (currNode != nullptr) {
-    currNode->init(initialize);
-    currNode = currNode->next;
+void InitOrDeinitGlobalVariables(int initialize, MemoryState* memory) {
+  InitNode* currentNode = initHeadNode;
+  while (currentNode != nullptr) {
+    currentNode->init(initialize, memory);
+    currentNode = currentNode->next;
   }
 }
 
@@ -98,23 +98,21 @@ RuntimeState* initRuntime() {
   if (firstRuntime) {
     isMainThread = 1;
     konan::consoleInit();
-
 #if KONAN_OBJC_INTEROP
     Kotlin_ObjCExport_initialize();
 #endif
-
-    InitOrDeinitGlobalVariables(INIT_GLOBALS);
+    InitOrDeinitGlobalVariables(INIT_GLOBALS, result->memoryState);
   }
-  InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS);
+  InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS, result->memoryState);
   return result;
 }
 
 void deinitRuntime(RuntimeState* state) {
   ResumeMemory(state->memoryState);
   bool lastRuntime = atomicAdd(&aliveRuntimesCount, -1) == 0;
-  InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS);
+  InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS, state->memoryState);
   if (lastRuntime)
-    InitOrDeinitGlobalVariables(DEINIT_GLOBALS);
+    InitOrDeinitGlobalVariables(DEINIT_GLOBALS, state->memoryState);
   WorkerDeinit(state->worker);
   DeinitMemory(state->memoryState);
   konanDestructInstance(state);
@@ -266,7 +264,8 @@ KBoolean Konan_Platform_isDebugBinary() {
 }
 
 void Kotlin_zeroOutTLSGlobals() {
-  InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS);
+  if (runtimeState != nullptr && runtimeState->memoryState != nullptr)
+    InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS, runtimeState->memoryState);
 }
 
 }  // extern "C"