diff --git a/compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java b/compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java index 333b2dde83b..11adbb23903 100644 --- a/compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java +++ b/compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java @@ -48736,6 +48736,16 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") diff --git a/compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt b/compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt new file mode 100644 index 00000000000..5729ac267ba --- /dev/null +++ b/compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt @@ -0,0 +1,41 @@ +// TARGET_BACKEND: NATIVE + +// FILE: 1.kt + +val O = if (true) "O" else "F" // to avoid const init +val K = if (true) "K" else "A" // to avoid const init + +// FILE: main.kt + +import kotlin.native.concurrent.* + +val sem = AtomicInt(0) + +fun box() : String { + val w1 = Worker.start() + val w2 = Worker.start() + val f1 = w1.execute( + mode = TransferMode.SAFE, + { }, + { + sem.increment(); + while (sem.value != 3) {} + O + } + ) + val f2 = w2.execute( + mode = TransferMode.SAFE, + { }, + { + sem.increment(); + while (sem.value != 3) {} + K + } + ) + while (sem.value != 2) {} + sem.value = 3 + val result = f1.result + f2.result + w1.requestTermination().result + w2.requestTermination().result + return result +} \ No newline at end of file diff --git a/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java b/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java index 1509dfc21fc..727da069d78 100644 --- a/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java +++ b/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java @@ -47308,6 +47308,16 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest { } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM, true); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") diff --git a/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java b/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java index 2fff5dd6b0d..c0b8b924f8a 100644 --- a/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java +++ b/compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java @@ -48736,6 +48736,16 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") diff --git a/compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java b/compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java index 88522c0bc89..bcaa380542d 100644 --- a/compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java +++ b/compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java @@ -38341,6 +38341,19 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes } } + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + @RunWith(JUnit3RunnerWithInners.class) + public static class TopLevelInitializtion extends AbstractLightAnalysisModeTest { + private void runTest(String testDataFilePath) throws Exception { + KotlinTestUtils.runTest(this::doTest, TargetBackend.JVM, testDataFilePath); + } + + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM, true); + } + } + @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") @RunWith(JUnit3RunnerWithInners.class) diff --git a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java index be23b0942f8..59375926e4f 100644 --- a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java +++ b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java @@ -35076,6 +35076,16 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest { } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JS, true); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") diff --git a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java index 5d050df95f6..453fad2b479 100644 --- a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java +++ b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java @@ -35208,6 +35208,16 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest { } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JS_IR, true); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") diff --git a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java index a1518a3f3b1..805610df799 100644 --- a/js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java +++ b/js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java @@ -31541,6 +31541,19 @@ public class IrCodegenBoxWasmTestGenerated extends AbstractIrCodegenBoxWasmTest } } + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + @RunWith(JUnit3RunnerWithInners.class) + public static class TopLevelInitializtion extends AbstractIrCodegenBoxWasmTest { + private void runTest(String testDataFilePath) throws Exception { + KotlinTestUtils.runTest0(this::doTest, TargetBackend.WASM, testDataFilePath); + } + + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^([^_](.+))\\.kt$"), null, TargetBackend.WASM, true); + } + } + @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT") @RunWith(JUnit3RunnerWithInners.class) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index ae3d34848cc..581ae745089 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -579,15 +579,21 @@ internal abstract class FunctionGenerationContext( fun param(index: Int): LLVMValueRef = LLVMGetParam(this.function, index)!! - fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef { - val result = LLVMBuildLoad(builder, value, name)!! + fun load(address: LLVMValueRef, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef { + val value = LLVMBuildLoad(builder, address, name)!! + if (memoryOrder != null) { + LLVMSetOrdering(value, memoryOrder) + } // Use loadSlot() API for that. assert(!isObjectRef(value)) - return result + return value } - fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = ""): LLVMValueRef { + fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef { val value = LLVMBuildLoad(builder, address, name)!! + if (memoryOrder != null) { + LLVMSetOrdering(value, memoryOrder) + } if (isObjectRef(value) && isVar) { val slot = resultSlot ?: alloca(LLVMTypeOf(value), variableLocation = null) storeStackRef(value, slot) @@ -1138,7 +1144,7 @@ internal abstract class FunctionGenerationContext( fun loadTypeInfo(objPtr: LLVMValueRef): LLVMValueRef { val typeInfoOrMetaPtr = structGep(objPtr, 0 /* typeInfoOrMeta_ */) - val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr) + val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire) // Clear two lower bits. val typeInfoOrMetaWithFlagsRaw = ptrToInt(typeInfoOrMetaWithFlags, codegen.intPtrType) val typeInfoOrMetaRaw = and(typeInfoOrMetaWithFlagsRaw, codegen.immTypeInfoMask) @@ -1317,7 +1323,7 @@ internal abstract class FunctionGenerationContext( } val bbInit = basicBlock("label_init", startLocationInfo, endLocationInfo) val bbExit = basicBlock("label_continue", startLocationInfo, endLocationInfo) - val objectVal = loadSlot(objectPtr, false) + val objectVal = loadSlot(objectPtr, false, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire) val objectInitialized = icmpUGt(ptrToInt(objectVal, codegen.intPtrType), codegen.immOneIntPtrType) val bbCurrent = currentBlock condBr(objectInitialized, bbExit, bbInit) diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 8ffb9c113b2..9a56b5c1908 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -499,21 +499,23 @@ ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) { #if KONAN_OBJC_INTEROP -void* ObjHeader::GetAssociatedObject() { - if (!has_meta_object()) { +void* ObjHeader::GetAssociatedObject() const { + auto metaObj = this->meta_object_or_null(); + if (metaObj == nullptr) { return nullptr; } - return this->meta_object()->associatedObject_; -} - -void** ObjHeader::GetAssociatedObjectLocation() { - return &this->meta_object()->associatedObject_; + return metaObj->associatedObject_; } void ObjHeader::SetAssociatedObject(void* obj) { this->meta_object()->associatedObject_ = obj; } +void* ObjHeader::CasAssociatedObject(void* expectedObj, void* obj) { + return __sync_val_compare_and_swap(&this->meta_object()->associatedObject_, expectedObj, obj); +} + + #endif // KONAN_OBJC_INTEROP class ForeignRefManager { diff --git a/kotlin-native/runtime/src/main/cpp/Atomic.h b/kotlin-native/runtime/src/main/cpp/Atomic.h index a40d24c023f..568cb81b448 100644 --- a/kotlin-native/runtime/src/main/cpp/Atomic.h +++ b/kotlin-native/runtime/src/main/cpp/Atomic.h @@ -12,33 +12,6 @@ ALWAYS_INLINE inline T atomicAdd(volatile T* where, T what) { #endif } -template -ALWAYS_INLINE inline T compareAndSwap(volatile T* where, T expectedValue, T newValue) { -#ifndef KONAN_NO_THREADS - return __sync_val_compare_and_swap(where, expectedValue, newValue); -#else - T oldValue = *where; - if (oldValue == expectedValue) { - *where = newValue; - } - return oldValue; -#endif -} - -template -ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T newValue) { -#ifndef KONAN_NO_THREADS - return __sync_bool_compare_and_swap(where, expectedValue, newValue); -#else - T oldValue = *where; - if (oldValue == expectedValue) { - *where = newValue; - return true; - } - return false; -#endif -} - #pragma clang diagnostic push #if (KONAN_ANDROID || KONAN_IOS || KONAN_WATCHOS || KONAN_LINUX) && (KONAN_ARM32 || KONAN_X86 || KONAN_MIPS32 || KONAN_MIPSEL32) @@ -49,26 +22,79 @@ ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T ne #pragma clang diagnostic ignored "-Watomic-alignment" #endif +// as if (std::atomic where).compare_exchange_strong(expectedValue, newValue) template +ALWAYS_INLINE inline bool compareExchange(volatile T& where, T &expectedValue, T newValue) { +#ifndef KONAN_NO_THREADS +#ifdef KONAN_NO_64BIT_ATOMIC + static_assert(sizeof(T) <= 4); +#endif + return __atomic_compare_exchange_n(&where, &expectedValue, newValue, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +#else + T oldValue = where; + if (oldValue == expectedValue) { + where = newValue; + return true; + } + expectedValue = oldValue; + return false; +#endif +} + +template +ALWAYS_INLINE inline T compareAndSwap(volatile T* where, T expectedValue, T newValue) { + compareExchange(*where, expectedValue, newValue); + return expectedValue; +} + +template +ALWAYS_INLINE inline bool compareAndSet(volatile T* where, T expectedValue, T newValue) { + return compareExchange(*where, expectedValue, newValue); +} + + +template ALWAYS_INLINE inline void atomicSet(volatile T* where, T what) { #ifndef KONAN_NO_THREADS - __atomic_store(where, &what, __ATOMIC_SEQ_CST); +#ifdef KONAN_NO_64BIT_ATOMIC + static_assert(sizeof(T) <= 4); +#endif + __atomic_store(where, &what, model); #else *where = what; #endif } template -ALWAYS_INLINE inline T atomicGet(volatile T* where) { +ALWAYS_INLINE inline void atomicSetRelease(volatile T* where, T what) { + return atomicSet<__ATOMIC_RELEASE>(where, what); +} + + +template +ALWAYS_INLINE inline T atomicGet(volatile const T* where) { #ifndef KONAN_NO_THREADS +#ifdef KONAN_NO_64BIT_ATOMIC + static_assert(sizeof(T) <= 4); +#endif T what; - __atomic_load(where, &what, __ATOMIC_SEQ_CST); + __atomic_load(where, &what, model); return what; #else return *where; #endif } +template +ALWAYS_INLINE inline T atomicGetAcquire(volatile const T* where) { + return atomicGet<__ATOMIC_ACQUIRE>(where); +} + +template +ALWAYS_INLINE inline T atomicGetRelaxed(volatile const T* where) { + return atomicGet<__ATOMIC_RELAXED>(where); +} + #pragma clang diagnostic pop static ALWAYS_INLINE inline void synchronize() { diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 0ffd021db63..84f4ab9500d 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -59,35 +59,38 @@ struct ObjHeader { } } + TypeInfo* typeInfoOrMetaRelaxed() const { return atomicGetRelaxed(&typeInfoOrMeta_);} + TypeInfo* typeInfoOrMetaAcquire() const { return atomicGetAcquire(&typeInfoOrMeta_);} + const TypeInfo* type_info() const { - return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_; + return clearPointerBits(typeInfoOrMetaAcquire(), OBJECT_TAG_MASK)->typeInfo_; } bool has_meta_object() const { - return AsMetaObject(typeInfoOrMeta_) != nullptr; + return meta_object_or_null() != nullptr; } MetaObjHeader* meta_object() { - if (auto* metaObject = AsMetaObject(typeInfoOrMeta_)) { + if (auto* metaObject = AsMetaObject(typeInfoOrMetaAcquire())) { return metaObject; } return createMetaObject(this); } - MetaObjHeader* meta_object_or_null() const noexcept { return AsMetaObject(typeInfoOrMeta_); } + MetaObjHeader* meta_object_or_null() const noexcept { return AsMetaObject(typeInfoOrMetaAcquire()); } ALWAYS_INLINE ObjHeader* GetWeakCounter(); ALWAYS_INLINE ObjHeader* GetOrSetWeakCounter(ObjHeader* counter); #ifdef KONAN_OBJC_INTEROP - ALWAYS_INLINE void* GetAssociatedObject(); - ALWAYS_INLINE void** GetAssociatedObjectLocation(); + ALWAYS_INLINE void* GetAssociatedObject() const; ALWAYS_INLINE void SetAssociatedObject(void* obj); + ALWAYS_INLINE void* CasAssociatedObject(void* expectedObj, void* obj); #endif inline bool local() const { - unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK); + unsigned bits = getPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK); return (bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) == (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER); } @@ -98,10 +101,10 @@ struct ObjHeader { const ArrayHeader* array() const { return reinterpret_cast(this); } inline bool permanent() const { - return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); + return hasPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_PERMANENT_CONTAINER); } - inline bool heap() const { return getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK) == 0; } + inline bool heap() const { return getPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK) == 0; } static MetaObjHeader* createMetaObject(ObjHeader* object); static void destroyMetaObject(ObjHeader* object); diff --git a/kotlin-native/runtime/src/main/cpp/ObjCExport.h b/kotlin-native/runtime/src/main/cpp/ObjCExport.h index 8fa2744ff6f..eb921a7a7b8 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjCExport.h +++ b/kotlin-native/runtime/src/main/cpp/ObjCExport.h @@ -23,8 +23,7 @@ inline static void SetAssociatedObject(ObjHeader* obj, id value) { } inline static id AtomicCompareAndSwapAssociatedObject(ObjHeader* obj, id expectedValue, id newValue) { - id* location = reinterpret_cast(obj->GetAssociatedObjectLocation()); - return __sync_val_compare_and_swap(location, expectedValue, newValue); + return static_cast(obj->CasAssociatedObject(expectedValue, newValue)); } inline static OBJ_GETTER(AllocInstanceWithAssociatedObject, const TypeInfo* typeInfo, id associatedObject) { diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp index 3cb6241e8b4..42f8258702c 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp +++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp @@ -472,25 +472,27 @@ RUNTIME_NOTHROW void Kotlin_initRuntimeIfNeededFromKotlin() { } } -} // extern "C" +static void CallInitGlobalAwaitInitialized(int *state) { + int localState; + // Switch to the native state to avoid dead-locks. + { + kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative); + do { + localState = atomicGetAcquire(state); + } while (localState != FILE_INITIALIZED && localState != FILE_FAILED_TO_INITIALIZE); + } + if (localState == FILE_FAILED_TO_INITIALIZE) ThrowFileFailedToInitializeException(); +} -namespace { -void callInitGlobalPossiblyLockImpl(int volatile* state, void (*init)()) { - int localState = *state; +NO_INLINE void CallInitGlobalPossiblyLock(int* state, void (*init)()) { + int localState = atomicGetAcquire(state); if (localState == FILE_INITIALIZED) return; if (localState == FILE_FAILED_TO_INITIALIZE) ThrowFileFailedToInitializeException(); int threadId = konan::currentThreadId(); if ((localState & 3) == FILE_BEING_INITIALIZED) { if ((localState & ~3) != (threadId << 2)) { - // Switch to the native state to avoid dead-locks. - kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative); - do { - localState = *state; - if (localState == FILE_FAILED_TO_INITIALIZE) - // Call of a Kotlin function. - kotlin::CallWithThreadState(ThrowFileFailedToInitializeException); - } while (localState != FILE_INITIALIZED); + CallInitGlobalAwaitInitialized(state); } return; } @@ -502,33 +504,15 @@ void callInitGlobalPossiblyLockImpl(int volatile* state, void (*init)()) { try { init(); } catch (...) { - *state = FILE_FAILED_TO_INITIALIZE; + atomicSetRelease(state, FILE_FAILED_TO_INITIALIZE); throw; } #endif - std::atomic_thread_fence(std::memory_order_release); - *state = FILE_INITIALIZED; + atomicSetRelease(state, FILE_INITIALIZED); } else { - // Switch to the native state to avoid dead-locks. - kotlin::ThreadStateGuard guard(kotlin::ThreadState::kNative); - do { - localState = *state; - if (localState == FILE_FAILED_TO_INITIALIZE) - // Call of a Kotlin function. - kotlin::CallWithThreadState(ThrowFileFailedToInitializeException); - } while (localState != FILE_INITIALIZED); + CallInitGlobalAwaitInitialized(state); } } -} - -extern "C" { - -NO_INLINE void CallInitGlobalPossiblyLock(int volatile* state, void (*init)()) { - callInitGlobalPossiblyLockImpl(state, init); - // Ensure proper synchronization around reading/writing of [state] (release barrier defined in callInitGlobalPossiblyLockImpl), - // also there is an acquire load of [state] in IrToBitcode.kt::evaluateFileGlobalInitializerCall. - std::atomic_thread_fence(std::memory_order_acquire); -} void CallInitThreadLocal(int volatile* globalState, int* localState, void (*init)()) { if (*localState == FILE_FAILED_TO_INITIALIZE || (globalState != nullptr && *globalState == FILE_FAILED_TO_INITIALIZE)) diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.h b/kotlin-native/runtime/src/main/cpp/Runtime.h index 835bf24b16c..218e8a13115 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.h +++ b/kotlin-native/runtime/src/main/cpp/Runtime.h @@ -38,7 +38,7 @@ void Kotlin_shutdownRuntime(); // Appends given node to an initializer list. void AppendToInitializersTail(struct InitNode*); -void CallInitGlobalPossiblyLock(int volatile* state, void (*init)()); +void CallInitGlobalPossiblyLock(int* state, void (*init)()); void CallInitThreadLocal(int volatile* globalState, int* localState, void (*init)()); bool Kotlin_memoryLeakCheckerEnabled(); diff --git a/kotlin-native/runtime/src/main/cpp/Worker.cpp b/kotlin-native/runtime/src/main/cpp/Worker.cpp index 4a66475d8f1..ba8b78576b6 100644 --- a/kotlin-native/runtime/src/main/cpp/Worker.cpp +++ b/kotlin-native/runtime/src/main/cpp/Worker.cpp @@ -324,8 +324,11 @@ class Future { void cancelUnlocked(MemoryState* memoryState); + KInt stateUnlocked() const { + Locker locker(&lock_); + return state_; + } // Those are called with the lock taken. - KInt state() const { return state_; } KInt id() const { return id_; } private: @@ -336,8 +339,8 @@ class Future { // Stable pointer with future's result. KNativePtr result_; // Lock and condition for waiting on the future. - pthread_mutex_t lock_; - pthread_cond_t cond_; + mutable pthread_mutex_t lock_; + mutable pthread_cond_t cond_; }; class State { @@ -488,7 +491,7 @@ class State { Locker locker(&lock_); auto it = futures_.find(id); if (it == futures_.end()) return INVALID; - return it->second->state(); + return it->second->stateUnlocked(); } OBJ_GETTER(consumeFutureUnlocked, KInt id) { diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index 7d7a7d1350a..c5a105d1d08 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -16,26 +16,12 @@ using namespace kotlin; -namespace { - -template -ALWAYS_INLINE T UnsafeRead(T* location) noexcept { -#if __has_feature(thread_sanitizer) - // Make TSAN think that this load is fine. - return __atomic_load_n(location, __ATOMIC_ACQUIRE); -#else - return *location; -#endif -} - -} // namespace - // static mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { // TODO: Consider extracting initialization scheme with speculative load. // `object->typeInfoOrMeta_` is assigned at most once. If we read some old value (i.e. not a meta object), // we will fail at CAS below. If we read the new value, we will immediately return it. - TypeInfo* typeInfo = UnsafeRead(&object->typeInfoOrMeta_); + TypeInfo* typeInfo = object->typeInfoOrMetaAcquire(); if (auto* metaObject = ObjHeader::AsMetaObject(typeInfo)) { return mm::ExtraObjectData::FromMetaObjHeader(metaObject); @@ -46,11 +32,10 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo); - TypeInfo* old = __sync_val_compare_and_swap(&object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data)); - if (old != typeInfo) { + if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { // Somebody else created `mm::ExtraObjectData` for this object mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, data); - return *reinterpret_cast(old); + return *reinterpret_cast(typeInfo); } return data; @@ -58,7 +43,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { void mm::ExtraObjectData::Uninstall() noexcept { auto *object = GetBaseObject(); - *const_cast(&object->typeInfoOrMeta_) = typeInfo_; + atomicSetRelease(const_cast(&object->typeInfoOrMeta_), typeInfo_); RuntimeAssert(!object->has_meta_object(), "Object has metaobject after removing metaobject"); #ifdef KONAN_OBJC_INTEROP diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp index 81763306fe9..9bb3918fba2 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp @@ -44,7 +44,7 @@ public: void Uninstall() noexcept; #ifdef KONAN_OBJC_INTEROP - void** GetAssociatedObjectLocation() noexcept { return &associatedObject_; } + std::atomic& AssociatedObject() noexcept { return associatedObject_; } #endif bool HasAssociatedObject() noexcept; void DetachAssociatedObject() noexcept; @@ -89,7 +89,7 @@ private: std::atomic flags_ = 0; #ifdef KONAN_OBJC_INTEROP - void* associatedObject_ = nullptr; + std::atomic associatedObject_ = nullptr; #endif std::atomic weakReferenceCounterOrBaseObject_; diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index aae4336d1c5..3e52eb3f6a2 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -58,19 +58,21 @@ ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) { #ifdef KONAN_OBJC_INTEROP -void* ObjHeader::GetAssociatedObject() { - if (!has_meta_object()) { +void* ObjHeader::GetAssociatedObject() const { + auto metaObject = meta_object_or_null(); + if (metaObject == nullptr) { return nullptr; } - return *GetAssociatedObjectLocation(); -} - -void** ObjHeader::GetAssociatedObjectLocation() { - return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetAssociatedObjectLocation(); + return mm::ExtraObjectData::FromMetaObjHeader(metaObject).AssociatedObject().load(std::memory_order_acquire); } void ObjHeader::SetAssociatedObject(void* obj) { - *GetAssociatedObjectLocation() = obj; + return mm::ExtraObjectData::FromMetaObjHeader(meta_object()).AssociatedObject().store(obj, std::memory_order_release); +} + +void* ObjHeader::CasAssociatedObject(void* expectedObj, void* obj) { + mm::ExtraObjectData::FromMetaObjHeader(meta_object()).AssociatedObject().compare_exchange_strong(expectedObj, obj); + return expectedObj; } #endif // KONAN_OBJC_INTEROP diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp index 03125de72ab..800d2e8f648 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp @@ -221,7 +221,7 @@ TEST_F(ThreadSuspensionTest, FileInitializationWithSuspend) { ASSERT_THAT(collectThreadData(), testing::IsEmpty()); ASSERT_FALSE(mm::IsThreadSuspensionRequested()); - volatile int lock = internal::FILE_NOT_INITIALIZED; + int lock = internal::FILE_NOT_INITIALIZED; auto scopedInitializationMock = ScopedInitializationMock(); EXPECT_CALL(*scopedInitializationMock, Call()).WillOnce([] { diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java index 55208e02bb1..13458bc2a92 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java @@ -38307,6 +38307,24 @@ public class NativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTest } } + @Nested + @TestMetadata("compiler/testData/codegen/box/topLevelInitializtion") + @TestDataPath("$PROJECT_ROOT") + @Tag("codegen") + @UseExtTestCaseGroupProvider() + public class TopLevelInitializtion { + @Test + public void testAllFilesPresentInTopLevelInitializtion() throws Exception { + KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/topLevelInitializtion"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true); + } + + @Test + @TestMetadata("concurrent.kt") + public void testConcurrent() throws Exception { + runTest("compiler/testData/codegen/box/topLevelInitializtion/concurrent.kt"); + } + } + @Nested @TestMetadata("compiler/testData/codegen/box/topLevelPrivate") @TestDataPath("$PROJECT_ROOT")