[K/N] Fix possible data races found by thread sanitizer

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java

@@ -48736,6 +48736,16 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
         }
     }
 
+    @Nested
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    public class TopLevelInitializtion {
+        @Test
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true);
+        }
+    }
+
     @Nested
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")

compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt

@@ -0,0 +1,41 @@
+// TARGET_BACKEND: NATIVE
+
+// FILE: 1.kt
+
+val O = if (true) "O" else "F" // to avoid const init
+val K = if (true) "K" else "A" // to avoid const init
+
+// FILE: main.kt
+
+import kotlin.native.concurrent.*
+
+val sem = AtomicInt(0)
+
+fun box() : String {
+    val w1 = Worker.start()
+    val w2 = Worker.start()
+    val f1 = w1.execute(
+        mode = TransferMode.SAFE,
+        { },
+        {
+            sem.increment();
+            while (sem.value != 3) {}
+            O
+        }
+    )
+    val f2 = w2.execute(
+        mode = TransferMode.SAFE,
+        { },
+        {
+            sem.increment();
+            while (sem.value != 3) {}
+            K
+        }
+    )
+    while (sem.value != 2) {}
+    sem.value = 3
+    val result = f1.result + f2.result
+    w1.requestTermination().result
+    w2.requestTermination().result
+    return result
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java

@@ -47308,6 +47308,16 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
         }
     }
 
+    @Nested
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    public class TopLevelInitializtion {
+        @Test
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM, true);
+        }
+    }
+
     @Nested
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java

@@ -48736,6 +48736,16 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
         }
     }
 
+    @Nested
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    public class TopLevelInitializtion {
+        @Test
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true);
+        }
+    }
+
     @Nested
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")

compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -38341,6 +38341,19 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
         }
     }
 
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    @RunWith(JUnit3RunnerWithInners.class)
+    public static class TopLevelInitializtion extends AbstractLightAnalysisModeTest {
+        private void runTest(String testDataFilePath) throws Exception {
+            KotlinTestUtils.runTest(this::doTest, TargetBackend.JVM, testDataFilePath);
+        }
+
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM, true);
+        }
+    }
+
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")
     @RunWith(JUnit3RunnerWithInners.class)

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java

@@ -35076,6 +35076,16 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
         }
     }
 
+    @Nested
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    public class TopLevelInitializtion {
+        @Test
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JS, true);
+        }
+    }
+
     @Nested
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java

@@ -35208,6 +35208,16 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest {
         }
     }
 
+    @Nested
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    public class TopLevelInitializtion {
+        @Test
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JS_IR, true);
+        }
+    }
+
     @Nested
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")

js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java

@@ -31541,6 +31541,19 @@ public class IrCodegenBoxWasmTestGenerated extends AbstractIrCodegenBoxWasmTest
         }
     }
 
+    @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+    @TestDataPath("$PROJECT_ROOT")
+    @RunWith(JUnit3RunnerWithInners.class)
+    public static class TopLevelInitializtion extends AbstractIrCodegenBoxWasmTest {
+        private void runTest(String testDataFilePath) throws Exception {
+            KotlinTestUtils.runTest0(this::doTest, TargetBackend.WASM, testDataFilePath);
+        }
+
+        public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^([^_](.+))\\.kt$"), null, TargetBackend.WASM, true);
+        }
+    }
+
     @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
     @TestDataPath("$PROJECT_ROOT")
     @RunWith(JUnit3RunnerWithInners.class)

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -579,15 +579,21 @@ internal abstract class FunctionGenerationContext(
 
     fun param(index: Int): LLVMValueRef = LLVMGetParam(this.function, index)!!
 
-    fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef {
+    fun load(address: LLVMValueRef, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef {
-        val result = LLVMBuildLoad(builder, value, name)!!
+        val value = LLVMBuildLoad(builder, address, name)!!
+        if (memoryOrder != null) {
+            LLVMSetOrdering(value, memoryOrder)
+        }
         // Use loadSlot() API for that.
         assert(!isObjectRef(value))
-        return result
+        return value
     }
 
-    fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = ""): LLVMValueRef {
+    fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef {
         val value = LLVMBuildLoad(builder, address, name)!!
+        if (memoryOrder != null) {
+            LLVMSetOrdering(value, memoryOrder)
+        }
         if (isObjectRef(value) && isVar) {
             val slot = resultSlot ?: alloca(LLVMTypeOf(value), variableLocation = null)
             storeStackRef(value, slot)
@@ -1138,7 +1144,7 @@ internal abstract class FunctionGenerationContext(
 
     fun loadTypeInfo(objPtr: LLVMValueRef): LLVMValueRef {
         val typeInfoOrMetaPtr = structGep(objPtr, 0  /* typeInfoOrMeta_ */)
-        val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr)
+        val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire)
         // Clear two lower bits.
         val typeInfoOrMetaWithFlagsRaw = ptrToInt(typeInfoOrMetaWithFlags, codegen.intPtrType)
         val typeInfoOrMetaRaw = and(typeInfoOrMetaWithFlagsRaw, codegen.immTypeInfoMask)
@@ -1317,7 +1323,7 @@ internal abstract class FunctionGenerationContext(
         }
         val bbInit = basicBlock("label_init", startLocationInfo, endLocationInfo)
         val bbExit = basicBlock("label_continue", startLocationInfo, endLocationInfo)
-        val objectVal = loadSlot(objectPtr, false)
+        val objectVal = loadSlot(objectPtr, false, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire)
         val objectInitialized = icmpUGt(ptrToInt(objectVal, codegen.intPtrType), codegen.immOneIntPtrType)
         val bbCurrent = currentBlock
         condBr(objectInitialized, bbExit, bbInit)

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -499,21 +499,23 @@ ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) {
 
 #if KONAN_OBJC_INTEROP
 
-void* ObjHeader::GetAssociatedObject() {
+void* ObjHeader::GetAssociatedObject() const {
-    if (!has_meta_object()) {
+    auto metaObj = this->meta_object_or_null();
+    if (metaObj == nullptr) {
       return nullptr;
     }
-    return this->meta_object()->associatedObject_;
+    return metaObj->associatedObject_;
-}
-
-void** ObjHeader::GetAssociatedObjectLocation() {
-    return &this->meta_object()->associatedObject_;
 }
 
 void ObjHeader::SetAssociatedObject(void* obj) {
     this->meta_object()->associatedObject_ = obj;
 }
 
+void* ObjHeader::CasAssociatedObject(void* expectedObj, void* obj) {
+    return __sync_val_compare_and_swap(&this->meta_object()->associatedObject_, expectedObj, obj);
+}
+
+
 #endif // KONAN_OBJC_INTEROP
 
 class ForeignRefManager {

kotlin-native/runtime/src/main/cpp/Atomic.h

@@ -12,33 +12,6 @@ ALWAYS_INLINE inline T atomicAdd(volatile T* where, T what) {
 #endif
 }
 
-template <typename T>
-ALWAYS_INLINE inline T compareAndSwap(volatile T* where, T expectedValue, T newValue) {
-#ifndef KONAN_NO_THREADS
-  return __sync_val_compare_and_swap(where, expectedValue, newValue);
-#else
-   T oldValue = *where;
-   if (oldValue == expectedValue) {
-        *where = newValue;
-   }
-   return oldValue;
-#endif
-}
-
-template <typename T>
-ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T newValue) {
-#ifndef KONAN_NO_THREADS
-  return __sync_bool_compare_and_swap(where, expectedValue, newValue);
-#else
-   T oldValue = *where;
-   if (oldValue == expectedValue) {
-        *where = newValue;
-        return true;
-   }
-   return false;
-#endif
-}
-
 #pragma clang diagnostic push
 
 #if (KONAN_ANDROID || KONAN_IOS || KONAN_WATCHOS || KONAN_LINUX) && (KONAN_ARM32 || KONAN_X86 || KONAN_MIPS32 || KONAN_MIPSEL32)
@@ -49,26 +22,79 @@ ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T ne
 #pragma clang diagnostic ignored "-Watomic-alignment"
 #endif
 
+// as if (std::atomic<T> where).compare_exchange_strong(expectedValue, newValue)
 template <typename T>
+ALWAYS_INLINE inline bool compareExchange(volatile T& where, T &expectedValue, T newValue) {
+#ifndef KONAN_NO_THREADS
+#ifdef KONAN_NO_64BIT_ATOMIC
+    static_assert(sizeof(T) <= 4);
+#endif
+    return __atomic_compare_exchange_n(&where, &expectedValue, newValue, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+#else
+    T oldValue = where;
+    if (oldValue == expectedValue) {
+        where = newValue;
+        return true;
+    }
+    expectedValue = oldValue;
+    return false;
+#endif
+}
+
+template <typename T>
+ALWAYS_INLINE inline T compareAndSwap(volatile T* where, T expectedValue, T newValue) {
+    compareExchange(*where, expectedValue, newValue);
+    return expectedValue;
+}
+
+template <typename T>
+ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T newValue) {
+    return compareExchange(*where, expectedValue, newValue);
+}
+
+
+template <int model = __ATOMIC_SEQ_CST, typename T>
 ALWAYS_INLINE inline void atomicSet(volatile T* where, T what) {
 #ifndef KONAN_NO_THREADS
-  __atomic_store(where, &what, __ATOMIC_SEQ_CST);
+#ifdef KONAN_NO_64BIT_ATOMIC
+  static_assert(sizeof(T) <= 4);
+#endif
+  __atomic_store(where, &what, model);
 #else
   *where = what;
 #endif
 }
 
 template <typename T>
-ALWAYS_INLINE inline T atomicGet(volatile T* where) {
+ALWAYS_INLINE inline void atomicSetRelease(volatile T* where, T what) {
+    return atomicSet<__ATOMIC_RELEASE>(where, what);
+}
+
+
+template <int model = __ATOMIC_SEQ_CST, typename T>
+ALWAYS_INLINE inline T atomicGet(volatile const T* where) {
 #ifndef KONAN_NO_THREADS
+#ifdef KONAN_NO_64BIT_ATOMIC
+  static_assert(sizeof(T) <= 4);
+#endif
   T what;
-  __atomic_load(where, &what, __ATOMIC_SEQ_CST);
+  __atomic_load(where, &what, model);
   return what;
 #else
   return *where;
 #endif
 }
 
+template <typename T>
+ALWAYS_INLINE inline T atomicGetAcquire(volatile const T* where) {
+    return atomicGet<__ATOMIC_ACQUIRE>(where);
+}
+
+template <typename T>
+ALWAYS_INLINE inline T atomicGetRelaxed(volatile const T* where) {
+    return atomicGet<__ATOMIC_RELAXED>(where);
+}
+
 #pragma clang diagnostic pop
 
 static ALWAYS_INLINE inline void synchronize() {

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -59,35 +59,38 @@ struct ObjHeader {
       }
   }
 
+  TypeInfo* typeInfoOrMetaRelaxed() const { return atomicGetRelaxed(&typeInfoOrMeta_);}
+  TypeInfo* typeInfoOrMetaAcquire() const { return atomicGetAcquire(&typeInfoOrMeta_);}
+
   const TypeInfo* type_info() const {
-    return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
+    return clearPointerBits(typeInfoOrMetaAcquire(), OBJECT_TAG_MASK)->typeInfo_;
   }
 
   bool has_meta_object() const {
-      return AsMetaObject(typeInfoOrMeta_) != nullptr;
+      return meta_object_or_null() != nullptr;
   }
 
   MetaObjHeader* meta_object() {
-      if (auto* metaObject = AsMetaObject(typeInfoOrMeta_)) {
+      if (auto* metaObject = AsMetaObject(typeInfoOrMetaAcquire())) {
           return metaObject;
       }
       return createMetaObject(this);
   }
 
-  MetaObjHeader* meta_object_or_null() const noexcept { return AsMetaObject(typeInfoOrMeta_); }
+  MetaObjHeader* meta_object_or_null() const noexcept { return AsMetaObject(typeInfoOrMetaAcquire()); }
 
   ALWAYS_INLINE ObjHeader* GetWeakCounter();
   ALWAYS_INLINE ObjHeader* GetOrSetWeakCounter(ObjHeader* counter);
 
 
 #ifdef KONAN_OBJC_INTEROP
-  ALWAYS_INLINE void* GetAssociatedObject();
+  ALWAYS_INLINE void* GetAssociatedObject() const;
-  ALWAYS_INLINE void** GetAssociatedObjectLocation();
   ALWAYS_INLINE void SetAssociatedObject(void* obj);
+  ALWAYS_INLINE void* CasAssociatedObject(void* expectedObj, void* obj);
 #endif
 
   inline bool local() const {
-    unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
+    unsigned bits = getPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK);
     return (bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) ==
         (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER);
   }
@@ -98,10 +101,10 @@ struct ObjHeader {
   const ArrayHeader* array() const { return reinterpret_cast<const ArrayHeader*>(this); }
 
   inline bool permanent() const {
-    return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
+    return hasPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_PERMANENT_CONTAINER);
   }
 
-  inline bool heap() const { return getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK) == 0; }
+  inline bool heap() const { return getPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK) == 0; }
 
   static MetaObjHeader* createMetaObject(ObjHeader* object);
   static void destroyMetaObject(ObjHeader* object);

kotlin-native/runtime/src/main/cpp/ObjCExport.h

@@ -23,8 +23,7 @@ inline static void SetAssociatedObject(ObjHeader* obj, id value) {
 }
 
 inline static id AtomicCompareAndSwapAssociatedObject(ObjHeader* obj, id expectedValue, id newValue) {
-    id* location = reinterpret_cast<id*>(obj->GetAssociatedObjectLocation());
+    return static_cast<id>(obj->CasAssociatedObject(expectedValue, newValue));
-    return __sync_val_compare_and_swap(location, expectedValue, newValue);
 }
 
 inline static OBJ_GETTER(AllocInstanceWithAssociatedObject, const TypeInfo* typeInfo, id associatedObject) {

kotlin-native/runtime/src/main/cpp/Runtime.cpp

@@ -472,25 +472,27 @@ RUNTIME_NOTHROW void Kotlin_initRuntimeIfNeededFromKotlin() {
     }
 }
 
-}  // extern "C"
+static void CallInitGlobalAwaitInitialized(int *state) {
+    int localState;
+    // Switch to the native state to avoid dead-locks.
+    {
+        kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative);
+        do {
+            localState = atomicGetAcquire(state);
+        } while (localState != FILE_INITIALIZED && localState != FILE_FAILED_TO_INITIALIZE);
+    }
+    if (localState == FILE_FAILED_TO_INITIALIZE) ThrowFileFailedToInitializeException();
+}
 
-namespace {
+NO_INLINE void CallInitGlobalPossiblyLock(int* state, void (*init)()) {
-void callInitGlobalPossiblyLockImpl(int volatile* state, void (*init)()) {
+    int localState = atomicGetAcquire(state);
-    int localState = *state;
     if (localState == FILE_INITIALIZED) return;
     if (localState == FILE_FAILED_TO_INITIALIZE)
         ThrowFileFailedToInitializeException();
     int threadId = konan::currentThreadId();
     if ((localState & 3) == FILE_BEING_INITIALIZED) {
         if ((localState & ~3) != (threadId << 2)) {
-            // Switch to the native state to avoid dead-locks.
+            CallInitGlobalAwaitInitialized(state);
-            kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative);
-            do {
-                localState = *state;
-                if (localState == FILE_FAILED_TO_INITIALIZE)
-                    // Call of a Kotlin function.
-                    kotlin::CallWithThreadState<kotlin::ThreadState::kRunnable>(ThrowFileFailedToInitializeException);
-            } while (localState != FILE_INITIALIZED);
         }
         return;
     }
@@ -502,33 +504,15 @@ void callInitGlobalPossiblyLockImpl(int volatile* state, void (*init)()) {
         try {
             init();
         } catch (...) {
-            *state = FILE_FAILED_TO_INITIALIZE;
+            atomicSetRelease(state, FILE_FAILED_TO_INITIALIZE);
             throw;
         }
 #endif
-        std::atomic_thread_fence(std::memory_order_release);
+        atomicSetRelease(state, FILE_INITIALIZED);
-        *state = FILE_INITIALIZED;
     } else {
-        // Switch to the native state to avoid dead-locks.
+        CallInitGlobalAwaitInitialized(state);
-        kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative);
-        do {
-            localState = *state;
-            if (localState == FILE_FAILED_TO_INITIALIZE)
-                // Call of a Kotlin function.
-                kotlin::CallWithThreadState<kotlin::ThreadState::kRunnable>(ThrowFileFailedToInitializeException);
-        } while (localState != FILE_INITIALIZED);
     }
 }
-}
-
-extern "C" {
-
-NO_INLINE void CallInitGlobalPossiblyLock(int volatile* state, void (*init)()) {
-    callInitGlobalPossiblyLockImpl(state, init);
-    // Ensure proper synchronization around reading/writing of [state] (release barrier defined in callInitGlobalPossiblyLockImpl),
-    // also there is an acquire load of [state] in IrToBitcode.kt::evaluateFileGlobalInitializerCall.
-    std::atomic_thread_fence(std::memory_order_acquire);
-}
 
 void CallInitThreadLocal(int volatile* globalState, int* localState, void (*init)()) {
     if (*localState == FILE_FAILED_TO_INITIALIZE || (globalState != nullptr && *globalState == FILE_FAILED_TO_INITIALIZE))

kotlin-native/runtime/src/main/cpp/Runtime.h

@@ -38,7 +38,7 @@ void Kotlin_shutdownRuntime();
 // Appends given node to an initializer list.
 void AppendToInitializersTail(struct InitNode*);
 
-void CallInitGlobalPossiblyLock(int volatile* state, void (*init)());
+void CallInitGlobalPossiblyLock(int* state, void (*init)());
 void CallInitThreadLocal(int volatile* globalState, int* localState, void (*init)());
 
 bool Kotlin_memoryLeakCheckerEnabled();

kotlin-native/runtime/src/main/cpp/Worker.cpp

@@ -324,8 +324,11 @@ class Future {
 
   void cancelUnlocked(MemoryState* memoryState);
 
+  KInt stateUnlocked() const {
+      Locker locker(&lock_);
+      return state_;
+  }
   // Those are called with the lock taken.
-  KInt state() const { return state_; }
   KInt id() const { return id_; }
 
  private:
@@ -336,8 +339,8 @@ class Future {
   // Stable pointer with future's result.
   KNativePtr result_;
   // Lock and condition for waiting on the future.
-  pthread_mutex_t lock_;
+  mutable pthread_mutex_t lock_;
-  pthread_cond_t cond_;
+  mutable pthread_cond_t cond_;
 };
 
 class State {
@@ -488,7 +491,7 @@ class State {
     Locker locker(&lock_);
     auto it = futures_.find(id);
     if (it == futures_.end()) return INVALID;
-    return it->second->state();
+    return it->second->stateUnlocked();
   }
 
   OBJ_GETTER(consumeFutureUnlocked, KInt id) {

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp

@@ -16,26 +16,12 @@
 
 using namespace kotlin;
 
-namespace {
-
-template <typename T>
-ALWAYS_INLINE T UnsafeRead(T* location) noexcept {
-#if __has_feature(thread_sanitizer)
-    // Make TSAN think that this load is fine.
-    return __atomic_load_n(location, __ATOMIC_ACQUIRE);
-#else
-    return *location;
-#endif
-}
-
-} // namespace
-
 // static
 mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
     // TODO: Consider extracting initialization scheme with speculative load.
     // `object->typeInfoOrMeta_` is assigned at most once. If we read some old value (i.e. not a meta object),
     // we will fail at CAS below. If we read the new value, we will immediately return it.
-    TypeInfo* typeInfo = UnsafeRead(&object->typeInfoOrMeta_);
+    TypeInfo* typeInfo = object->typeInfoOrMetaAcquire();
 
     if (auto* metaObject = ObjHeader::AsMetaObject(typeInfo)) {
         return mm::ExtraObjectData::FromMetaObjHeader(metaObject);
@@ -46,11 +32,10 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
     auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo);
 
-    TypeInfo* old = __sync_val_compare_and_swap(&object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data));
+    if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
-    if (old != typeInfo) {
         // Somebody else created `mm::ExtraObjectData` for this object
         mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, data);
-        return *reinterpret_cast<mm::ExtraObjectData*>(old);
+        return *reinterpret_cast<mm::ExtraObjectData*>(typeInfo);
     }
 
     return data;
@@ -58,7 +43,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
 
 void mm::ExtraObjectData::Uninstall() noexcept {
     auto *object = GetBaseObject();
-    *const_cast<const TypeInfo**>(&object->typeInfoOrMeta_) = typeInfo_;
+    atomicSetRelease(const_cast<const TypeInfo**>(&object->typeInfoOrMeta_), typeInfo_);
     RuntimeAssert(!object->has_meta_object(), "Object has metaobject after removing metaobject");
 
 #ifdef KONAN_OBJC_INTEROP

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp

@@ -44,7 +44,7 @@ public:
     void Uninstall() noexcept;
 
 #ifdef KONAN_OBJC_INTEROP
-    void** GetAssociatedObjectLocation() noexcept { return &associatedObject_; }
+    std::atomic<void*>& AssociatedObject() noexcept { return associatedObject_; }
 #endif
     bool HasAssociatedObject() noexcept;
     void DetachAssociatedObject() noexcept;
@@ -89,7 +89,7 @@ private:
     std::atomic<uint32_t> flags_ = 0;
 
 #ifdef KONAN_OBJC_INTEROP
-    void* associatedObject_ = nullptr;
+    std::atomic<void*> associatedObject_ = nullptr;
 #endif
 
     std::atomic<ObjHeader*> weakReferenceCounterOrBaseObject_;

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -58,19 +58,21 @@ ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) {
 
 #ifdef KONAN_OBJC_INTEROP
 
-void* ObjHeader::GetAssociatedObject() {
+void* ObjHeader::GetAssociatedObject() const {
-    if (!has_meta_object()) {
+    auto metaObject = meta_object_or_null();
+    if (metaObject == nullptr) {
         return nullptr;
     }
-    return *GetAssociatedObjectLocation();
+    return mm::ExtraObjectData::FromMetaObjHeader(metaObject).AssociatedObject().load(std::memory_order_acquire);
-}
-
-void** ObjHeader::GetAssociatedObjectLocation() {
-    return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetAssociatedObjectLocation();
 }
 
 void ObjHeader::SetAssociatedObject(void* obj) {
-    *GetAssociatedObjectLocation() = obj;
+    return mm::ExtraObjectData::FromMetaObjHeader(meta_object()).AssociatedObject().store(obj, std::memory_order_release);
+}
+
+void* ObjHeader::CasAssociatedObject(void* expectedObj, void* obj) {
+    mm::ExtraObjectData::FromMetaObjHeader(meta_object()).AssociatedObject().compare_exchange_strong(expectedObj, obj);
+    return expectedObj;
 }
 
 #endif // KONAN_OBJC_INTEROP

kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp

@@ -221,7 +221,7 @@ TEST_F(ThreadSuspensionTest, FileInitializationWithSuspend) {
     ASSERT_THAT(collectThreadData(), testing::IsEmpty());
     ASSERT_FALSE(mm::IsThreadSuspensionRequested());
 
-    volatile int lock = internal::FILE_NOT_INITIALIZED;
+    int lock = internal::FILE_NOT_INITIALIZED;
 
     auto scopedInitializationMock = ScopedInitializationMock();
     EXPECT_CALL(*scopedInitializationMock, Call()).WillOnce([] {

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java

@@ -38307,6 +38307,24 @@ public class NativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTest
             }
         }
 
+        @Nested
+        @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion")
+        @TestDataPath("$PROJECT_ROOT")
+        @Tag("codegen")
+        @UseExtTestCaseGroupProvider()
+        public class TopLevelInitializtion {
+            @Test
+            public void testAllFilesPresentInTopLevelInitializtion() throws Exception {
+                KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true);
+            }
+
+            @Test
+            @TestMetadata("concurrent.kt")
+            public void testConcurrent() throws Exception {
+                runTest("compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt");
+            }
+        }
+
         @Nested
         @TestMetadata("compiler/testData/codegen/box/topLevelPrivate")
         @TestDataPath("$PROJECT_ROOT")