diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 152c95ddb7b..3dcf71366aa 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -921,7 +921,6 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, } val singleton = context.llvmDeclarations.forSingleton(irClass) val instanceAddress = singleton.instanceStorage - val instanceShadowAddress = singleton.instanceShadowStorage if (storageKind == ObjectStorageKind.PERMANENT) { return loadSlot(instanceAddress.getAddress(this), false) @@ -938,13 +937,13 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, val typeInfo = codegen.typeInfoForAllocation(irClass) val defaultConstructor = irClass.constructors.single { it.valueParameters.size == 0 } val ctor = codegen.llvmFunction(defaultConstructor) - val (initFunction, args) = + val initFunction = if (storageKind == ObjectStorageKind.SHARED && context.config.threadsAreAllowed) { - val shadowObjectPtr = instanceShadowAddress!!.getAddress(this) - context.llvm.initSharedInstanceFunction to listOf(objectPtr, shadowObjectPtr, typeInfo, ctor) + context.llvm.initSharedInstanceFunction } else { - context.llvm.initInstanceFunction to listOf(objectPtr, typeInfo, ctor) + context.llvm.initInstanceFunction } + val args = listOf(objectPtr, typeInfo, ctor) val newValue = call(initFunction, args, Lifetime.GLOBAL, exceptionHandler) val bbInitResult = currentBlock br(bbExit) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt index 60da5877af1..1893901b1df 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmDeclarations.kt @@ -67,7 +67,7 @@ internal class ClassLlvmDeclarations( val singletonDeclarations: SingletonLlvmDeclarations?, val objCDeclarations: KotlinObjCClassLlvmDeclarations?) -internal class SingletonLlvmDeclarations(val instanceStorage: AddressAccess, val instanceShadowStorage: AddressAccess?) +internal class SingletonLlvmDeclarations(val instanceStorage: AddressAccess) internal class KotlinObjCClassLlvmDeclarations( val classPointerGlobal: StaticData.Global, @@ -279,12 +279,7 @@ private class DeclarationsGeneratorVisitor(override val context: Context) : val symbolName = "kobjref:" + qualifyInternalName(irClass) val instanceAddress = addKotlinGlobal(symbolName, getLLVMType(irClass.defaultType), threadLocal = threadLocal) - val instanceShadowAddress = if (threadLocal || storageKind == ObjectStorageKind.PERMANENT) null else { - val shadowSymbolName = "kshadowobjref:" + qualifyInternalName(irClass) - addKotlinGlobal(shadowSymbolName, getLLVMType(irClass.defaultType), threadLocal = true) - } - - return SingletonLlvmDeclarations(instanceAddress, instanceShadowAddress) + return SingletonLlvmDeclarations(instanceAddress) } private fun createKotlinObjCClassDeclarations(irClass: IrClass): KotlinObjCClassLlvmDeclarations { diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 86fc53b1ebd..a51ad342add 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -451,6 +451,8 @@ struct MemoryState { uint64_t allocSinceLastGcThreshold; #endif // USE_GC + KStdUnorderedMap initializingSingletons; + #if COLLECT_STATISTIC #define CONTAINER_ALLOC_STAT(state, size, container) state->statistic.incAlloc(size, container); #define CONTAINER_DESTROY_STAT(state, container) \ @@ -1999,7 +2001,7 @@ OBJ_GETTER(initInstance, template OBJ_GETTER(initSharedInstance, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { #if KONAN_NO_THREADS ObjHeader* value = *location; if (value != nullptr) { @@ -2025,25 +2027,32 @@ OBJ_GETTER(initSharedInstance, } #endif // KONAN_NO_EXCEPTIONS #else // KONAN_NO_THREADS - ObjHeader* value = *localLocation; - if (value != nullptr) RETURN_OBJ(value); + auto it = memoryState->initializingSingletons.find(location); + if (it != memoryState->initializingSingletons.end()) { + RETURN_OBJ(it->second); + } ObjHeader* initializing = reinterpret_cast(1); // Spin lock. + ObjHeader* value = nullptr; while ((value = __sync_val_compare_and_swap(location, nullptr, initializing)) == initializing); if (value != nullptr) { // OK'ish, inited by someone else. RETURN_OBJ(value); } ObjHeader* object = AllocInstance(typeInfo, OBJ_RESULT); - UpdateHeapRef(localLocation, object); + auto insertIt = memoryState->initializingSingletons.insert({location, object}); + RuntimeCheck(insertIt.second, "object cannot be assigned twice into initializingSingletons"); + addHeapRef(object); #if KONAN_NO_EXCEPTIONS ctor(object); if (Strict) FreezeSubgraph(object); UpdateHeapRef(location, object); synchronize(); + memoryState->initializingSingletons.erase(location); + releaseHeapRef(object); return object; #else // KONAN_NO_EXCEPTIONS try { @@ -2052,11 +2061,14 @@ OBJ_GETTER(initSharedInstance, FreezeSubgraph(object); UpdateHeapRef(location, object); synchronize(); + memoryState->initializingSingletons.erase(location); + releaseHeapRef(object); return object; } catch (...) { UpdateReturnRef(OBJ_RESULT, nullptr); zeroHeapRef(location); - zeroHeapRef(localLocation); + memoryState->initializingSingletons.erase(location); + releaseHeapRef(object); synchronize(); throw; } @@ -2803,12 +2815,12 @@ OBJ_GETTER(InitInstanceRelaxed, } OBJ_GETTER(InitSharedInstanceStrict, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { - RETURN_RESULT_OF(initSharedInstance, location, localLocation, typeInfo, ctor); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initSharedInstance, location, typeInfo, ctor); } OBJ_GETTER(InitSharedInstanceRelaxed, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { - RETURN_RESULT_OF(initSharedInstance, location, localLocation, typeInfo, ctor); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initSharedInstance, location, typeInfo, ctor); } void SetStackRefStrict(ObjHeader** location, const ObjHeader* object) { diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 8fdb5803d7b..9768b5a1840 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -461,11 +461,11 @@ OBJ_GETTER(InitInstance, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); OBJ_GETTER(InitSharedInstanceStrict, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); OBJ_GETTER(InitSharedInstanceRelaxed, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); OBJ_GETTER(InitSharedInstance, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); // Weak reference operations. // Atomically clears counter object reference. diff --git a/runtime/src/relaxed/cpp/MemoryImpl.cpp b/runtime/src/relaxed/cpp/MemoryImpl.cpp index 0763899a16c..4cdbda5e3b9 100644 --- a/runtime/src/relaxed/cpp/MemoryImpl.cpp +++ b/runtime/src/relaxed/cpp/MemoryImpl.cpp @@ -25,8 +25,8 @@ OBJ_GETTER(InitInstance, } OBJ_GETTER(InitSharedInstance, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { - RETURN_RESULT_OF(InitSharedInstanceRelaxed, location, localLocation, typeInfo, ctor); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitSharedInstanceRelaxed, location, typeInfo, ctor); } void ReleaseHeapRef(const ObjHeader* object) { diff --git a/runtime/src/strict/cpp/MemoryImpl.cpp b/runtime/src/strict/cpp/MemoryImpl.cpp index 27f47b7e0c2..0a5a7afd0a8 100644 --- a/runtime/src/strict/cpp/MemoryImpl.cpp +++ b/runtime/src/strict/cpp/MemoryImpl.cpp @@ -25,8 +25,8 @@ OBJ_GETTER(InitInstance, } OBJ_GETTER(InitSharedInstance, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { - RETURN_RESULT_OF(InitSharedInstanceStrict, location, localLocation, typeInfo, ctor); + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitSharedInstanceStrict, location, typeInfo, ctor); } void ReleaseHeapRef(const ObjHeader* object) {