Freeze top-level val. (#1906)

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/KotlinCodeModel.kt

@@ -173,7 +173,6 @@ object KotlinTypes {
     val set by CollectionClassifier
     val map by CollectionClassifier
 
-
     val nativePtr by InteropType
 
     val cOpaque by InteropType
@@ -198,7 +197,6 @@ object KotlinTypes {
     val cFunction by InteropClassifier
 
     val objCObjectVar by InteropClassifier
-    val objCStringVarOf by InteropClassifier
 
     val objCObjectBase by InteropClassifier
     val objCObjectBaseMeta by InteropClassifier
@@ -220,6 +218,7 @@ object KotlinTypes {
 
     private object InteropClassifier : ClassifierAtPackage("kotlinx.cinterop")
     private object InteropType : TypeAtPackage("kotlinx.cinterop")
+
 }
 
 abstract class KotlinFile(

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -44,6 +44,7 @@ import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl
 import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection
 import org.jetbrains.kotlin.ir.types.typeWith
 import org.jetbrains.kotlin.ir.util.*
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.KotlinType
@@ -367,6 +368,9 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val
 
     val listOfInternal = internalFunction("listOfInternal")
 
+    val threadLocal =
+            context.builtIns.builtInsModule.findClassAcrossModuleDependencies(ClassId.topLevel(FqName("kotlin.native.ThreadLocal")))!!
+
     private fun internalFunction(name: String): IrSimpleFunctionSymbol =
             symbolTable.referenceSimpleFunction(context.getInternalFunctions(name).single())
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt

@@ -307,7 +307,7 @@ internal val ClassDescriptor.objectInstanceShadowFieldSymbolName: String
         assert (this.isExported())
         assert (this.kind.isSingleton)
         assert (!this.isUnit())
-        assert (this.symbol.objectIsShared)
+        assert (this.objectIsShared)
 
         return "kshadowobjref:$fqNameSafe"
     }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -77,7 +77,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
 
     fun getObjectInstanceShadowStorage(descriptor: ClassDescriptor): LLVMValueRef {
         assert (!descriptor.isUnit())
-        assert (descriptor.symbol.objectIsShared)
+        assert (descriptor.objectIsShared)
         val llvmGlobal = if (!isExternal(descriptor)) {
             context.llvmDeclarations.forSingleton(descriptor).instanceShadowFieldRef!!
         } else {
@@ -300,6 +300,11 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         }
     }
 
+    fun freeze(value: LLVMValueRef, exceptionHandler: ExceptionHandler) {
+        if (isObjectRef(value))
+            call(context.llvm.freezeSubgraph, listOf(value),  Lifetime.IRRELEVANT, exceptionHandler)
+    }
+
     private fun updateReturnRef(value: LLVMValueRef, address: LLVMValueRef) {
         call(context.llvm.updateReturnRefFunction, listOf(address, value))
     }
@@ -694,7 +699,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
             }
         }
 
-        val shared = descriptor.symbol.objectIsShared && context.config.threadsAreAllowed
+        val shared = descriptor.objectIsShared && context.config.threadsAreAllowed
         val objectPtr = codegen.getObjectInstanceStorage(descriptor, shared)
         val bbCurrent = currentBlock
         val bbInit= basicBlock("label_init", locationInfo)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -418,6 +418,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val appendToInitalizersTail = importRtFunction("AppendToInitializersTail")
     val initRuntimeIfNeeded = importRtFunction("Kotlin_initRuntimeIfNeeded")
     val mutationCheck = importRtFunction("MutationCheck")
+    val freezeSubgraph = importRtFunction("FreezeSubgraph")
 
     val createKotlinObjCClass by lazy { importRtFunction("CreateKotlinObjCClass") }
     val getObjCKotlinTypeInfo by lazy { importRtFunction("GetObjCKotlinTypeInfo") }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -50,9 +50,12 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameUnsafe
 
 private val threadLocalAnnotationFqName = FqName("kotlin.native.ThreadLocal")
 
-val IrClassSymbol.objectIsShared get() =
+val IrClass.objectIsShared get() =
     !descriptor.annotations.hasAnnotation(threadLocalAnnotationFqName)
 
+val IrField.isShared get() =
+    !descriptor.annotations.hasAnnotation(threadLocalAnnotationFqName) && !descriptor.isVar
+
 internal fun emitLLVM(context: Context, phaser: PhaseManager) {
     val irModule = context.irModule!!
 
@@ -373,14 +376,16 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     //-------------------------------------------------------------------------//
 
     val INIT_GLOBALS = 0
-    val DEINIT_THREAD_LOCAL_GLOBALS = 1
+    val INIT_THREAD_LOCAL_GLOBALS = 1
-    val DEINIT_GLOBALS = 2
+    val DEINIT_THREAD_LOCAL_GLOBALS = 2
+    val DEINIT_GLOBALS = 3
 
     private fun createInitBody(): LLVMValueRef {
         val initFunction = LLVMAddFunction(context.llvmModule, "", kInitFuncType)!!
         generateFunction(codegen, initFunction) {
             using(FunctionScope(initFunction, "init_body", it)) {
                 val bbInit = basicBlock("init", null)
+                val bbLocalInit = basicBlock("local_init", null)
                 val bbLocalDeinit = basicBlock("local_deinit", null)
                 val bbGlobalDeinit = basicBlock("global_deinit", null)
                 val bbDefault = basicBlock("default", null) {
@@ -389,6 +394,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
                 switch(LLVMGetParam(initFunction, 0)!!,
                         listOf(Int32(INIT_GLOBALS).llvm                to bbInit,
+                               Int32(INIT_THREAD_LOCAL_GLOBALS).llvm   to bbLocalInit,
                                Int32(DEINIT_THREAD_LOCAL_GLOBALS).llvm to bbLocalDeinit,
                                Int32(DEINIT_GLOBALS).llvm              to bbGlobalDeinit),
                         bbDefault)
@@ -398,9 +404,26 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
                     context.llvm.fileInitializers
                             .forEach {
                                 if (it.initializer?.expression !is IrConst<*>?) {
-                                    val initialization = evaluateExpression(it.initializer!!.expression)
+                                    if (it.isShared) {
-                                    val address = context.llvmDeclarations.forStaticField(it).storage
+                                        val initialization = evaluateExpression(it.initializer!!.expression)
-                                    storeAny(initialization, address)
+                                        val address = context.llvmDeclarations.forStaticField(it).storage
+                                        freeze(initialization, currentCodeContext.exceptionHandler)
+                                        storeAny(initialization, address)
+                                    }
+                                }
+                            }
+                    ret(null)
+                }
+
+                appendingTo(bbLocalInit) {
+                    context.llvm.fileInitializers
+                            .forEach {
+                                if (it.initializer?.expression !is IrConst<*>?) {
+                                   if (!it.isShared) {
+                                       val initialization = evaluateExpression(it.initializer!!.expression)
+                                       val address = context.llvmDeclarations.forStaticField(it).storage
+                                       storeAny(initialization, address)
+                                    }
                                 }
                             }
                     ret(null)
@@ -408,17 +431,25 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
                 appendingTo(bbLocalDeinit) {
                     context.llvm.fileInitializers.forEach {
-                        val descriptor = it
+                        // Only if a subject for memory management.
-                        if (!descriptor.type.binaryTypeIsReference())
+                        if (it.type.binaryTypeIsReference() && !it.isShared) {
-                            return@forEach // Is not a subject for memory management.
+                            val address = context.llvmDeclarations.forStaticField(it).storage
-                        val address = context.llvmDeclarations.forStaticField(descriptor).storage
+                            storeAny(codegen.kNullObjHeaderPtr, address)
-                        storeAny(codegen.kNullObjHeaderPtr, address)
+                        }
                     }
                     context.llvm.objects.forEach { storeAny(codegen.kNullObjHeaderPtr, it) }
                     ret(null)
                 }
 
                 appendingTo(bbGlobalDeinit) {
+                    context.llvm.fileInitializers
+                            // Only if a subject for memory management.
+                            .forEach {
+                                if (it.type.binaryTypeIsReference() && it.isShared) {
+                                    val address = context.llvmDeclarations.forStaticField(it).storage
+                                    storeAny(codegen.kNullObjHeaderPtr, address)
+                                }
+                            }
                     context.llvm.sharedObjects.forEach { storeAny(codegen.kNullObjHeaderPtr, it) }
                     ret(null)
                 }
@@ -1426,8 +1457,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             }
             functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner))
         } else {
-            assert (value.receiver == null)
+            assert(value.receiver == null)
             val globalValue = context.llvmDeclarations.forStaticField(value.symbol.owner).storage
+            if (value.symbol.owner.isShared)
+                functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler)
             functionGenerationContext.storeAny(valueToAssign, globalValue)
         }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt

@@ -330,7 +330,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) :
         } else {
             "kobjref:" + qualifyInternalName(descriptor)
         }
-        val threadLocal = !(descriptor.symbol.objectIsShared && context.config.threadsAreAllowed)
+        val threadLocal = !(descriptor.objectIsShared && context.config.threadsAreAllowed)
         val instanceFieldRef = addGlobal(
                 symbolName, getLLVMType(descriptor.defaultType), isExported = isExported, threadLocal = threadLocal)
 
@@ -390,7 +390,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) :
             val name = "kvar:" + qualifyInternalName(descriptor)
 
             val storage = addGlobal(
-                    name, getLLVMType(descriptor.type), isExported = false, threadLocal = true)
+                    name, getLLVMType(descriptor.type), isExported = false, threadLocal = !declaration.isShared)
 
             this.staticFields[descriptor] = StaticFieldLlvmDeclarations(storage)
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.backend.konan.llvm
 
 import llvm.*
+import org.jetbrains.kotlin.backend.common.ir.ir2string
 import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
 import org.jetbrains.kotlin.ir.IrElement
@@ -121,7 +122,7 @@ internal class VariableManager(val functionGenerationContext: FunctionGeneration
 
     internal fun createImmutable(descriptor: ValueDescriptor, value: LLVMValueRef) : Int {
         if (contextVariablesToIndex.containsKey(descriptor))
-            throw Error("$descriptor is already defined")
+            throw Error("${ir2string(descriptor)} is already defined")
         val index = variables.size
         variables.add(ValueRecord(value, descriptor.name))
         contextVariablesToIndex[descriptor] = index

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/InteropLowering.kt

@@ -69,7 +69,7 @@ internal class InteropLoweringPart1(val context: Context) : IrBuildingTransforme
         currentFile = irFile
         irFile.transformChildrenVoid(this)
 
-        topLevelInitializers.forEach { irFile.addTopLevelInitializer(it) }
+        topLevelInitializers.forEach { irFile.addTopLevelInitializer(it, context, false) }
         topLevelInitializers.clear()
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/TestProcessor.kt

@@ -565,8 +565,8 @@ internal class TestProcessor (val context: KonanBackendContext) {
                 irFile.addChild(ir)
                 val irConstructor = ir.constructors.single()
                 irFile.addTopLevelInitializer(
-                        IrCallImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, irConstructor.returnType, irConstructor.symbol)
+                        IrCallImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, irConstructor.returnType, irConstructor.symbol),
-                )
+                        context, threadLocal = true)
             }
 
     /** Check if this fqName already used or not. */
@@ -587,6 +587,8 @@ internal class TestProcessor (val context: KonanBackendContext) {
             return
         }
 
+        // TODO: an awful hack, we make this initializer thread local, so that it doesn't freeze suite,
+        // and later on we could modify some suite's properties. This shall be redesigned.
         irFile.addTopLevelInitializer(builder.irBlock {
             val constructorCall = irCall(symbols.topLevelSuiteConstructor).apply {
                 putValueArgument(0, IrConstImpl.string(UNDEFINED_OFFSET, UNDEFINED_OFFSET,
@@ -597,7 +599,7 @@ internal class TestProcessor (val context: KonanBackendContext) {
                     symbols.topLevelSuiteRegisterTestCase,
                     symbols.topLevelSuiteRegisterFunction,
                     functions)
-        })
+        }, context, threadLocal = true)
     }
 
     private fun createTestSuites(irFile: IrFile, annotationCollector: AnnotationCollector) {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/ir/util/IrUtils2.kt

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.ir.util
 
 import org.jetbrains.kotlin.backend.common.CommonBackendContext
 import org.jetbrains.kotlin.backend.common.descriptors.substitute
+import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.KonanBackendContext
 import org.jetbrains.kotlin.backend.konan.KonanCompilationException
 import org.jetbrains.kotlin.backend.konan.descriptors.synthesizedName
@@ -27,7 +28,9 @@ import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.descriptors.FunctionDescriptor
 import org.jetbrains.kotlin.descriptors.ParameterDescriptor
 import org.jetbrains.kotlin.descriptors.ValueParameterDescriptor
+import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptorImpl
 import org.jetbrains.kotlin.descriptors.annotations.Annotations
+import org.jetbrains.kotlin.descriptors.annotations.AnnotationsImpl
 import org.jetbrains.kotlin.descriptors.impl.PropertyDescriptorImpl
 import org.jetbrains.kotlin.descriptors.impl.ValueParameterDescriptorImpl
 import org.jetbrains.kotlin.ir.IrElement
@@ -47,7 +50,6 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.OverridingStrategy
 import org.jetbrains.kotlin.resolve.OverridingUtil
-import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.typeUtil.immediateSupertypes
@@ -58,10 +60,14 @@ internal fun IrExpression.isNullConst() = this is IrConst<*> && this.kind == IrC
 
 private var topLevelInitializersCounter = 0
 
-internal fun IrFile.addTopLevelInitializer(expression: IrExpression) {
+internal fun IrFile.addTopLevelInitializer(expression: IrExpression, context: KonanBackendContext, threadLocal: Boolean) {
     val fieldDescriptor = PropertyDescriptorImpl.create(
             this.packageFragmentDescriptor,
-            Annotations.EMPTY,
+            if (threadLocal)
+                AnnotationsImpl(listOf(AnnotationDescriptorImpl(context.ir.symbols.threadLocal.defaultType,
+                        emptyMap(), SourceElement.NO_SOURCE)))
+            else
+                Annotations.EMPTY,
             Modality.FINAL,
             Visibilities.PRIVATE,
             false,
@@ -76,7 +82,6 @@ internal fun IrFile.addTopLevelInitializer(expression: IrExpression) {
             false
     )
 
-    val builtIns = fieldDescriptor.builtIns
     fieldDescriptor.setType(expression.type.toKotlinType(), emptyList(), null, null as KotlinType?)
     fieldDescriptor.initialize(null, null)
 
@@ -123,8 +128,8 @@ private fun createFakeOverride(
         is PropertyDescriptor ->
             IrPropertyImpl(startOffset, endOffset, IrDeclarationOrigin.FAKE_OVERRIDE, descriptor).apply {
                 // TODO: add field if getter is missing?
-                getter = descriptor.getter?.createFunction() as IrSimpleFunction?
+                getter = descriptor.getter?.createFunction()
-                setter = descriptor.setter?.createFunction() as IrSimpleFunction?
+                setter = descriptor.setter?.createFunction()
             }
         else -> TODO(descriptor.toString())
     }
@@ -238,7 +243,7 @@ fun IrSimpleFunction.setOverrides(symbolTable: ReferenceSymbolTable) {
 fun IrClass.simpleFunctions(): List<IrSimpleFunction> = this.declarations.flatMap {
     when (it) {
         is IrSimpleFunction -> listOf(it)
-        is IrProperty -> listOfNotNull(it.getter as IrSimpleFunction?, it.setter as IrSimpleFunction?)
+        is IrProperty -> listOfNotNull(it.getter, it.setter)
         else -> emptyList()
     }
 }

backend.native/tests/runtime/memory/escape2.kt

@@ -19,6 +19,7 @@ fun bar(b: B) {
     foo(c)
 }
 
+@ThreadLocal
 val global = B()
 
 @Test fun runTest() {

backend.native/tests/runtime/workers/freeze_stress.kt

@@ -68,6 +68,7 @@ class Graph(val nodes: List<Node>, val roots: List<Node>)
 fun min(x: Int, y: Int) = if (x < y) x else y
 fun max(x: Int, y: Int) = if (x > y) x else y
 
+@ThreadLocal
 val random = Random(42)
 
 fun generate(condensationSize: Int, branchingFactor: Int, swellingFactor: Int): Graph {

runtime/src/main/cpp/Memory.cpp

@@ -1667,7 +1667,7 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
  * Theory of operations.
  *
  * Kotlin/Native supports object graph freezing, allowing to make certain subgraph immutable and thus
- * suitable for safe sharing amongs multiple concurrent executors. This operation recursively operates
+ * suitable for safe sharing amongst multiple concurrent executors. This operation recursively operates
  * on all objects reachable from the given object, and marks them as frozen. In frozen state object's
  * fields cannot be modified, and so, lifetime of frozen objects correlates. Practically, it means
  * that lifetimes of all strongly connected components are fully controlled by incoming reference
@@ -1687,6 +1687,7 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
  * references could be passed across multiple threads.
  */
 void FreezeSubgraph(ObjHeader* root) {
+  if (root == nullptr) return;
   // First check that passed object graph has no cycles.
   // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir.
   ContainerHeader* rootContainer = root->container();

runtime/src/main/cpp/Runtime.cpp

@@ -39,8 +39,9 @@ InitNode* initTailNode = nullptr;
 
 enum {
   INIT_GLOBALS = 0,
-  DEINIT_THREAD_LOCAL_GLOBALS = 1,
+  INIT_THREAD_LOCAL_GLOBALS = 1,
-  DEINIT_GLOBALS = 2
+  DEINIT_THREAD_LOCAL_GLOBALS = 2,
+  DEINIT_GLOBALS = 3
 };
 
 enum {
@@ -78,10 +79,13 @@ RuntimeState* initRuntime() {
   RuntimeState* result = konanConstructInstance<RuntimeState>();
   if (!result) return nullptr;
   result->memoryState = InitMemory();
+  bool firstRuntime = atomicAdd(&aliveRuntimesCount, 1) == 1;
   // Keep global variables in state as well.
-  InitOrDeinitGlobalVariables(INIT_GLOBALS);
+  if (firstRuntime) {
-  konan::consoleInit();
+    konan::consoleInit();
-  atomicAdd(&aliveRuntimesCount, 1);
+    InitOrDeinitGlobalVariables(INIT_GLOBALS);
+  }
+  InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS);
   return result;
 }
 

runtime/src/main/kotlin/kotlin/native/test/Launcher.kt

@@ -18,6 +18,7 @@ package kotlin.native.test
 
 import kotlin.system.exitProcess
 
+@ThreadLocal
 private val _generatedSuites = mutableListOf<TestSuite>()
 
 internal fun registerSuite(suite: TestSuite): Unit {

runtime/src/main/kotlin/kotlin/native/worker/Atomics.kt

@@ -173,9 +173,19 @@ class AtomicReference<T>(private var value: T? = null) {
     external public fun get(): T?
 }
 
-internal object UNINITIALIZED
+internal object UNINITIALIZED {
+    // So that single-threaded configs can use those as well.
+    init {
+        freeze()
+    }
+}
 
-internal object INITIALIZING
+internal object INITIALIZING {
+    // So that single-threaded configs can use those as well.
+    init {
+        freeze()
+    }
+}
 
 @Frozen
 internal class AtomicLazyImpl<out T>(initializer: () -> T) : Lazy<T> {