From 9dea3497520d1c1427ddaf882cffb5fdf09dbc90 Mon Sep 17 00:00:00 2001 From: Pavel Kunyavskiy Date: Mon, 14 Nov 2022 17:08:28 +0100 Subject: [PATCH] [K/N] Support of @Volatile annotation ^KT-54944 --- .../jetbrains/kotlin/backend/konan/Context.kt | 6 + .../kotlin/backend/konan/KonanFqNames.kt | 1 + .../backend/konan/KonanLoweringPhases.kt | 10 +- .../kotlin/backend/konan/ToplevelPhases.kt | 1 + .../jetbrains/kotlin/backend/konan/ir/Ir.kt | 3 + .../backend/konan/llvm/CodeGenerator.kt | 46 ++-- .../kotlin/backend/konan/llvm/ContextUtils.kt | 6 + .../backend/konan/llvm/IntrinsicGenerator.kt | 83 ++++++- .../kotlin/backend/konan/llvm/IrToBitcode.kt | 79 ++++--- .../konan/lower/VolatileFieldsLowering.kt | 171 ++++++++++++++ .../backend/konan/optimizations/DFGBuilder.kt | 85 ++++--- .../cpp/CompilerCInterface.cpp | 4 + .../runtime/src/legacymm/cpp/Memory.cpp | 14 ++ kotlin-native/runtime/src/main/cpp/Atomic.cpp | 147 ------------ kotlin-native/runtime/src/main/cpp/Memory.h | 6 + .../kotlin/native/concurrent/Atomics.kt | 211 +++++++++++++----- .../kotlin/native/internal/IntrinsicType.kt | 10 + kotlin-native/runtime/src/mm/cpp/Memory.cpp | 16 ++ .../runtime/src/mm/cpp/ObjectOps.cpp | 24 +- .../runtime/src/mm/cpp/ObjectOps.hpp | 3 + 20 files changed, 631 insertions(+), 295 deletions(-) create mode 100644 kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/VolatileFieldsLowering.kt diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 5663580b27d..84292be0b49 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -50,6 +50,10 @@ import kotlin.LazyThreadSafetyMode.PUBLICATION internal class NativeMapping : DefaultMapping() { data class BridgeKey(val target: IrSimpleFunction, val bridgeDirections: BridgeDirections) + enum class AtomicFunctionType { + COMPARE_AND_SWAP, COMPARE_AND_SET, GET_AND_SET, GET_AND_ADD; + } + data class AtomicFunctionKey(val field: IrField, val type: AtomicFunctionType) val outerThisFields = DefaultDelegateFactory.newDeclarationToDeclarationMapping() val enumValueGetters = DefaultDelegateFactory.newDeclarationToDeclarationMapping() @@ -62,6 +66,8 @@ internal class NativeMapping : DefaultMapping() { val boxFunctions = DefaultDelegateFactory.newDeclarationToDeclarationMapping() val unboxFunctions = DefaultDelegateFactory.newDeclarationToDeclarationMapping() val loweredInlineClassConstructors = DefaultDelegateFactory.newDeclarationToDeclarationMapping() + val volatileFieldToAtomicFunction = mutableMapOf() + val functionToVolatileField = DefaultDelegateFactory.newDeclarationToDeclarationMapping() } internal class Context( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt index 4b90637d80e..5db525e2ce2 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt @@ -25,6 +25,7 @@ object KonanFqNames { val cancellationException = FqName("kotlin.coroutines.cancellation.CancellationException") val threadLocal = FqName("kotlin.native.concurrent.ThreadLocal") val sharedImmutable = FqName("kotlin.native.concurrent.SharedImmutable") + val volatile = FqName("kotlin.concurrent.Volatile") val frozen = FqName("kotlin.native.internal.Frozen") val frozenLegacyMM = FqName("kotlin.native.internal.FrozenLegacyMM") val leakDetectorCandidate = FqName("kotlin.native.internal.LeakDetectorCandidate") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLoweringPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLoweringPhases.kt index 09f31907207..eaeafae87db 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLoweringPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLoweringPhases.kt @@ -274,6 +274,13 @@ internal val tailrecPhase = makeKonanFileLoweringPhase( prerequisite = setOf(localFunctionsPhase) ) +internal val volatilePhase = makeKonanFileLoweringPhase( + ::VolatileFieldsLowering, + name = "VolatileFields", + description = "Volatile fields processing", + prerequisite = setOf(localFunctionsPhase) +) + internal val defaultParameterExtentPhase = makeKonanFileOpPhase( { context, irFile -> KonanDefaultArgumentStubGenerator(context).lower(irFile) @@ -335,7 +342,8 @@ internal val testProcessorPhase = makeKonanFileOpPhase( internal val delegationPhase = makeKonanFileLoweringPhase( { PropertyDelegationLowering(it.generationState) }, name = "Delegation", - description = "Delegation lowering" + description = "Delegation lowering", + prerequisite = setOf(volatilePhase) ) internal val functionReferencePhase = makeKonanFileLoweringPhase( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt index 11deaa20d8c..33c4f5af382 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt @@ -165,6 +165,7 @@ internal val allLoweringsPhase = SameTypeNamedCompilerPhase( enumConstructorsPhase, initializersPhase, localFunctionsPhase, + volatilePhase, tailrecPhase, defaultParameterExtentPhase, innerClassPhase, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index d719cb3dbb3..aa0c511cf2f 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -99,6 +99,7 @@ internal abstract class KonanSymbols( val symbolName = topLevelClass(RuntimeNames.symbolNameAnnotation) val filterExceptions = topLevelClass(RuntimeNames.filterExceptions) val exportForCppRuntime = topLevelClass(RuntimeNames.exportForCppRuntime) + val typedIntrinsic = topLevelClass(RuntimeNames.typedIntrinsicAnnotation) val objCMethodImp = symbolTable.referenceClass(descriptorsLookup.interopBuiltIns.objCMethodImp) @@ -415,6 +416,8 @@ internal abstract class KonanSymbols( val sharedImmutable = topLevelClass(KonanFqNames.sharedImmutable) + val volatile = topLevelClass(KonanFqNames.volatile) + val eagerInitialization = topLevelClass(KonanFqNames.eagerInitialization) private fun topLevelClass(fqName: FqName): IrClassSymbol = irBuiltIns.findClass(fqName.shortName(), fqName.parent())!! diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 54027a4886e..b6c4c67ab85 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -560,21 +560,22 @@ internal abstract class FunctionGenerationContext( fun param(index: Int): LLVMValueRef = LLVMGetParam(this.function, index)!! - fun load(address: LLVMValueRef, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef { + fun load(address: LLVMValueRef, name: String = "", + memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null + ): LLVMValueRef { val value = LLVMBuildLoad(builder, address, name)!! - if (memoryOrder != null) { - LLVMSetOrdering(value, memoryOrder) - } + memoryOrder?.let { LLVMSetOrdering(value, it) } + alignment?.let { LLVMSetAlignment(value, it) } // Use loadSlot() API for that. assert(!isObjectRef(value)) return value } - fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef { + fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "", + memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null): LLVMValueRef { val value = LLVMBuildLoad(builder, address, name)!! - if (memoryOrder != null) { - LLVMSetOrdering(value, memoryOrder) - } + memoryOrder?.let { LLVMSetOrdering(value, it) } + alignment?.let { LLVMSetAlignment(value, it) } if (isObjectRef(value) && isVar) { val slot = resultSlot ?: alloca(LLVMTypeOf(value), variableLocation = null) storeStackRef(value, slot) @@ -582,8 +583,10 @@ internal abstract class FunctionGenerationContext( return value } - fun store(value: LLVMValueRef, ptr: LLVMValueRef) { - LLVMBuildStore(builder, value, ptr) + fun store(value: LLVMValueRef, ptr: LLVMValueRef, memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null) { + val store = LLVMBuildStore(builder, value, ptr) + memoryOrder?.let { LLVMSetOrdering(store, it) } + alignment?.let { LLVMSetAlignment(store, it) } } fun storeHeapRef(value: LLVMValueRef, ptr: LLVMValueRef) { @@ -594,12 +597,12 @@ internal abstract class FunctionGenerationContext( updateRef(value, ptr, onStack = true) } - fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean) = if (isObjectRef(value)) { - if (onStack) storeStackRef(value, ptr) else storeHeapRef(value, ptr) - null - } else { - LLVMBuildStore(builder, value, ptr) + fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean, isVolatile: Boolean = false, alignment: Int? = null) { + when { + isObjectRef(value) -> updateRef(value, ptr, onStack, isVolatile, alignment) + else -> store(value, ptr, if (isVolatile) LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent else null, alignment) } + } fun freeze(value: LLVMValueRef, exceptionHandler: ExceptionHandler) { if (isObjectRef(value)) @@ -618,14 +621,21 @@ internal abstract class FunctionGenerationContext( call(llvm.updateReturnRefFunction, listOf(address, value)) } - private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean) { + private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean, + isVolatile: Boolean = false, alignment: Int? = null) { + require(alignment == null || alignment == runtime.pointerAlignment) if (onStack) { + require(!isVolatile) { "Stack ref update can't be volatile"} if (context.memoryModel == MemoryModel.STRICT) store(value, address) else call(llvm.updateStackRefFunction, listOf(address, value)) } else { - call(llvm.updateHeapRefFunction, listOf(address, value)) + if (isVolatile && context.memoryModel == MemoryModel.EXPERIMENTAL) { + call(llvm.UpdateVolatileHeapRef, listOf(address, value)) + } else { + call(llvm.updateHeapRefFunction, listOf(address, value)) + } } } @@ -1349,7 +1359,7 @@ internal abstract class FunctionGenerationContext( } addPhiIncoming(slotsPhi!!, prologueBb to slots) memScoped { - slotToVariableLocation.forEach { slot, variable -> + slotToVariableLocation.forEach { (slot, variable) -> val expr = longArrayOf(DwarfOp.DW_OP_plus_uconst.value, runtime.pointerSize * slot.toLong()).toCValues() DIInsertDeclaration( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 807aa00f4f2..d3b39c4c300 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -457,6 +457,12 @@ internal class Llvm(private val generationState: NativeGenerationState, val modu val Kotlin_processFieldInMark by lazyRtFunction val Kotlin_processEmptyObjectInMark by lazyRtFunction + val UpdateVolatileHeapRef by lazyRtFunction + val CompareAndSetVolatileHeapRef by lazyRtFunction + val CompareAndSwapVolatileHeapRef by lazyRtFunction + val GetAndSetVolatileHeapRef by lazyRtFunction + + val tlsMode by lazy { when (target) { KonanTarget.WASM32, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt index 190afeef8bd..b8864dc2981 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt @@ -12,6 +12,7 @@ import org.jetbrains.kotlin.backend.konan.llvm.objc.genObjCSelector import org.jetbrains.kotlin.backend.konan.reportCompilationError import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrField import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.symbols.IrConstructorSymbol @@ -90,7 +91,16 @@ internal enum class IntrinsicType { INTEROP_FUNPTR_INVOKE, INTEROP_MEMORY_COPY, // Worker - WORKER_EXECUTE + WORKER_EXECUTE, + // Atomics + COMPARE_AND_SET_FIELD, + COMPARE_AND_SWAP_FIELD, + GET_AND_SET_FIELD, + GET_AND_ADD_FIELD, + COMPARE_AND_SET, + COMPARE_AND_SWAP, + GET_AND_SET, + GET_AND_ADD, } internal enum class ConstantConstructorIntrinsicType { @@ -117,6 +127,8 @@ internal interface IntrinsicGeneratorEnvironment { fun evaluateExplicitArgs(expression: IrFunctionAccessExpression): List fun evaluateExpression(value: IrExpression, resultSlot: LLVMValueRef?): LLVMValueRef + + fun getObjectFieldPointer(thisRef: LLVMValueRef, field: IrField): LLVMValueRef } internal fun tryGetIntrinsicType(callSite: IrFunctionAccessExpression): IntrinsicType? = @@ -242,6 +254,10 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv IntrinsicType.INTEROP_MEMORY_COPY -> emitMemoryCopy(callSite, args) IntrinsicType.IS_EXPERIMENTAL_MM -> emitIsExperimentalMM() IntrinsicType.THE_UNIT_INSTANCE -> theUnitInstanceRef.llvm + IntrinsicType.COMPARE_AND_SET -> emitCompareAndSet(callSite, args) + IntrinsicType.COMPARE_AND_SWAP -> emitCompareAndSwap(callSite, args, resultSlot) + IntrinsicType.GET_AND_SET -> emitGetAndSet(callSite, args, resultSlot) + IntrinsicType.GET_AND_ADD -> emitGetAndAdd(callSite, args) IntrinsicType.GET_CONTINUATION, IntrinsicType.RETURN_IF_SUSPENDED, IntrinsicType.INTEROP_BITS_TO_FLOAT, @@ -253,7 +269,11 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv IntrinsicType.INTEROP_CONVERT, IntrinsicType.ENUM_VALUES, IntrinsicType.ENUM_VALUE_OF, - IntrinsicType.WORKER_EXECUTE -> + IntrinsicType.WORKER_EXECUTE, + IntrinsicType.COMPARE_AND_SET_FIELD, + IntrinsicType.COMPARE_AND_SWAP_FIELD, + IntrinsicType.GET_AND_SET_FIELD, + IntrinsicType.GET_AND_ADD_FIELD -> reportNonLoweredIntrinsic(intrinsicType) IntrinsicType.INIT_INSTANCE, IntrinsicType.OBJC_INIT_BY, @@ -292,6 +312,65 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv private fun FunctionGenerationContext.emitIsExperimentalMM(): LLVMValueRef = llvm.int1(context.memoryModel == MemoryModel.EXPERIMENTAL) + // cmpxcgh llvm instruction return pair. idnex is index of required element of this pair + enum class CmpExchangeMode(val index:Int) { + SWAP(0), + SET(1) + } + + private fun FunctionGenerationContext.emitCmpExchange(callSite: IrCall, args: List, mode: CmpExchangeMode, resultSlot: LLVMValueRef?): LLVMValueRef { + val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!! + require(args.size == 3) + val address = environment.getObjectFieldPointer(args[0], field) + return if (isObjectRef(args[1])) { + require(context.memoryModel == MemoryModel.EXPERIMENTAL) + when (mode) { + CmpExchangeMode.SET -> call(llvm.CompareAndSetVolatileHeapRef, listOf(address, args[1], args[2])) + CmpExchangeMode.SWAP -> call(llvm.CompareAndSwapVolatileHeapRef, listOf(address, args[1], args[2]), + environment.calculateLifetime(callSite), resultSlot = resultSlot) + } + } else { + val cmp = LLVMBuildAtomicCmpXchg(builder, address, args[1], args[2], + LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent, + LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent, + SingleThread = 0 + )!! + + LLVMBuildExtractValue(builder, cmp, mode.index, "")!! + } + } + + private fun FunctionGenerationContext.emitAtomicRMW(callSite: IrCall, args: List, op: LLVMAtomicRMWBinOp, resultSlot: LLVMValueRef?): LLVMValueRef { + val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!! + require(args.size == 2) + val address = environment.getObjectFieldPointer(args[0], field) + return if (isObjectRef(args[1])) { + require(op == LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg) + require(context.memoryModel == MemoryModel.EXPERIMENTAL) + call(llvm.GetAndSetVolatileHeapRef, listOf(address, args[1]), + environment.calculateLifetime(callSite), resultSlot = resultSlot) + } else { + LLVMBuildAtomicRMW(builder, op, address, args[1], + LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent, + singleThread = 0 + )!! + } + } + + private fun FunctionGenerationContext.emitCompareAndSet(callSite: IrCall, args: List): LLVMValueRef { + return emitCmpExchange(callSite, args, CmpExchangeMode.SET, null) + } + private fun FunctionGenerationContext.emitCompareAndSwap(callSite: IrCall, args: List, resultSlot: LLVMValueRef?): LLVMValueRef { + return emitCmpExchange(callSite, args, CmpExchangeMode.SWAP, resultSlot) + } + private fun FunctionGenerationContext.emitGetAndSet(callSite: IrCall, args: List, resultSlot: LLVMValueRef?): LLVMValueRef { + return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg, resultSlot) + } + private fun FunctionGenerationContext.emitGetAndAdd(callSite: IrCall, args: List): LLVMValueRef { + return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpAdd, null) + } + + private fun FunctionGenerationContext.emitGetNativeNullPtr(): LLVMValueRef = llvm.kNullInt8Ptr diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 3c830867fee..71615820def 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -228,6 +228,9 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, override fun evaluateExpression(value: IrExpression, resultSlot: LLVMValueRef?): LLVMValueRef = this@CodeGeneratorVisitor.evaluateExpression(value, resultSlot) + + override fun getObjectFieldPointer(thisRef: LLVMValueRef, field: IrField): LLVMValueRef = + this@CodeGeneratorVisitor.fieldPtrOfClass(thisRef, field) } private val intrinsicGenerator = IntrinsicGenerator(intrinsicGeneratorEnvironment) @@ -1660,35 +1663,46 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, private fun evaluateGetField(value: IrGetField, resultSlot: LLVMValueRef?): LLVMValueRef { context.log { "evaluateGetField : ${ir2string(value)}" } - return if (!value.symbol.owner.isStatic) { - val thisPtr = evaluateExpression(value.receiver!!) - functionGenerationContext.loadSlot( - fieldPtrOfClass(thisPtr, value.symbol.owner), !value.symbol.owner.isFinal, resultSlot) - } else { - assert(value.receiver == null) - if (value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true) { - evaluateConst(value.symbol.owner.initializer?.expression as IrConst<*>).llvm - } else { + val alignment = when { + value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8 + else -> null + } + val order = when { + value.symbol.owner.hasAnnotation(KonanFqNames.volatile) -> + LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent + else -> null + } + + val fieldAddress = when { + !value.symbol.owner.isStatic -> { + fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner) + } + value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true -> { + // TODO: probably can be removed, as they are inlined. + return evaluateConst(value.symbol.owner.initializer?.expression as IrConst<*>).llvm + } + else -> { if (context.config.threadsAreAllowed && value.symbol.owner.isGlobalNonPrimitive(context)) { functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) } - val ptr = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress( - functionGenerationContext - ) - functionGenerationContext.loadSlot(ptr, !value.symbol.owner.isFinal, resultSlot) + generationState.llvmDeclarations + .forStaticField(value.symbol.owner) + .storageAddressAccess + .getAddress(functionGenerationContext) } - }.also { - if (value.type.classifierOrNull?.isClassWithFqName(vectorType) == true) - LLVMSetAlignment(it, 8) - } + return functionGenerationContext.loadSlot( + fieldAddress, !value.symbol.owner.isFinal, resultSlot, + memoryOrder = order, + alignment = alignment + ) } //-------------------------------------------------------------------------// - private fun needMutationCheck(irClass: IrClass): Boolean { + private fun needMutationCheck(irField: IrField): Boolean { // For now we omit mutation checks on immutable types, as this allows initialization in constructor // and it is assumed that API doesn't allow to change them. - return context.config.freezing.enableFreezeChecks && !irClass.isFrozen(context) + return context.config.freezing.enableFreezeChecks && !irField.parentAsClass.isFrozen(context) && !irField.hasAnnotation(KonanFqNames.volatile) } private fun needLifetimeConstraintsCheck(valueToAssign: LLVMValueRef, irClass: IrClass): Boolean { @@ -1724,13 +1738,14 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, } val valueToAssign = evaluateExpression(value.value) - val store = if (!value.symbol.owner.isStatic) { + val address: LLVMValueRef + if (!value.symbol.owner.isStatic) { val thisPtr = evaluateExpression(value.receiver!!) assert(thisPtr.type == codegen.kObjHeaderPtr) { LLVMPrintTypeToString(thisPtr.type)?.toKString().toString() } val parentAsClass = value.symbol.owner.parentAsClass - if (needMutationCheck(parentAsClass)) { + if (needMutationCheck(value.symbol.owner)) { functionGenerationContext.call(llvm.mutationCheck, listOf(functionGenerationContext.bitcast(codegen.kObjHeaderPtr, thisPtr)), Lifetime.IRRELEVANT, currentCodeContext.exceptionHandler) @@ -1738,21 +1753,26 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, if (needLifetimeConstraintsCheck(valueToAssign, parentAsClass)) { functionGenerationContext.call(llvm.checkLifetimesConstraint, listOf(thisPtr, valueToAssign)) } - functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner), false) + address = fieldPtrOfClass(thisPtr, value.symbol.owner) } else { assert(value.receiver == null) - val globalAddress = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress( - functionGenerationContext - ) if (context.config.threadsAreAllowed && value.symbol.owner.storageKind(context) == FieldStorageKind.GLOBAL) functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler) if (value.symbol.owner.shouldBeFrozen(context) && value.origin != ObjectClassLowering.IrStatementOriginFieldPreInit) functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler) - functionGenerationContext.storeAny(valueToAssign, globalAddress, false) + address = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress( + functionGenerationContext + ) } - if (store != null && value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true) { - LLVMSetAlignment(store, 8) + val alignment = when { + value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8 + else -> null } + functionGenerationContext.storeAny( + valueToAssign, address, false, + isVolatile = value.symbol.owner.hasAnnotation(KonanFqNames.volatile), + alignment = alignment, + ) assert (value.type.isUnit()) return codegen.theUnitInstanceRef.llvm @@ -2379,8 +2399,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState, val bbExit = basicBlock("label_continue", null) moveBlockAfterEntry(bbExit) moveBlockAfterEntry(bbInit) - val state = load(statePtr) - LLVMSetOrdering(state, LLVMAtomicOrdering.LLVMAtomicOrderingAcquire) + val state = load(statePtr, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire) condBr(icmpEq(state, llvm.int32(FILE_INITIALIZED)), bbExit, bbInit) positionAtEnd(bbInit) call(llvm.callInitGlobalPossiblyLock, listOf(statePtr, initializerPtr), diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/VolatileFieldsLowering.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/VolatileFieldsLowering.kt new file mode 100644 index 00000000000..a09dc558802 --- /dev/null +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/VolatileFieldsLowering.kt @@ -0,0 +1,171 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.konan.lower + +import org.jetbrains.kotlin.backend.common.FileLoweringPass +import org.jetbrains.kotlin.backend.common.ir.addDispatchReceiver +import org.jetbrains.kotlin.backend.common.lower.* +import org.jetbrains.kotlin.backend.konan.* +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.backend.konan.NativeMapping +import org.jetbrains.kotlin.backend.konan.ir.buildSimpleAnnotation +import org.jetbrains.kotlin.backend.konan.llvm.IntrinsicType +import org.jetbrains.kotlin.backend.konan.llvm.tryGetIntrinsicType +import org.jetbrains.kotlin.ir.IrStatement +import org.jetbrains.kotlin.ir.builders.declarations.addValueParameter +import org.jetbrains.kotlin.ir.builders.declarations.buildFun +import org.jetbrains.kotlin.ir.builders.irString +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.expressions.* +import org.jetbrains.kotlin.ir.types.IrType +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.visitors.* +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.util.capitalizeDecapitalize.* + +object IR_DECLARATION_ORIGIN_VOLATILE : IrDeclarationOriginImpl("VOLATILE") + +internal class VolatileFieldsLowering(val context: Context) : FileLoweringPass { + private fun buildIntrinsicFunction(irField: IrField, intrinsicType: IntrinsicType, builder: IrSimpleFunction.() -> Unit) = context.irFactory.buildFun { + isExternal = true + origin = IR_DECLARATION_ORIGIN_VOLATILE + name = Name.special("<${intrinsicType.name.decapitalizeSmart()}-${irField.name}>") + startOffset = irField.startOffset + endOffset = irField.endOffset + }.apply { + val parentClass = irField.parents.filterIsInstance().first() + parent = parentClass + addDispatchReceiver { + startOffset = irField.startOffset + endOffset = irField.endOffset + type = parentClass.defaultType + } + builder() + annotations += buildSimpleAnnotation(context.irBuiltIns, + SYNTHETIC_OFFSET, SYNTHETIC_OFFSET, + context.ir.symbols.typedIntrinsic.owner, intrinsicType.name) + } + + private fun buildCasFunction(irField: IrField, intrinsicType: IntrinsicType, functionReturnType: IrType) = + buildIntrinsicFunction(irField, intrinsicType ) { + returnType = functionReturnType + addValueParameter { + startOffset = irField.startOffset + endOffset = irField.endOffset + name = Name.identifier("expectedValue") + type = irField.type + } + addValueParameter { + startOffset = irField.startOffset + endOffset = irField.endOffset + name = Name.identifier("newValue") + type = irField.type + } + } + + private fun buildAtomicRWMFunction(irField: IrField, intrinsicType: IntrinsicType) = + buildIntrinsicFunction(irField, intrinsicType) { + returnType = irField.type + addValueParameter { + startOffset = irField.startOffset + endOffset = irField.endOffset + name = Name.identifier("value") + type = irField.type + } + } + + + private inline fun atomicFunction(irField: IrField, type: NativeMapping.AtomicFunctionType, builder: () -> IrSimpleFunction): IrSimpleFunction { + val key = NativeMapping.AtomicFunctionKey(irField, type) + return context.mapping.volatileFieldToAtomicFunction.getOrPut(key) { + builder().also { + context.mapping.functionToVolatileField[it] = irField + } + } + } + + private fun compareAndSetFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.COMPARE_AND_SET) { + this.buildCasFunction(irField, IntrinsicType.COMPARE_AND_SET, this.context.irBuiltIns.booleanType) + } + private fun compareAndSwapFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.COMPARE_AND_SWAP) { + this.buildCasFunction(irField, IntrinsicType.COMPARE_AND_SWAP, irField.type) + } + private fun getAndSetFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.GET_AND_SET) { + this.buildAtomicRWMFunction(irField, IntrinsicType.GET_AND_SET) + } + private fun getAndAddFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.GET_AND_ADD) { + this.buildAtomicRWMFunction(irField, IntrinsicType.GET_AND_ADD) + } + + private fun IrField.isInteger() = type == context.irBuiltIns.intType || + type == context.irBuiltIns.longType || + type == context.irBuiltIns.shortType || + type == context.irBuiltIns.byteType + + override fun lower(irFile: IrFile) { + irFile.transformChildrenVoid(object : IrBuildingTransformer(context) { + override fun visitClass(declaration: IrClass): IrStatement { + declaration.declarations.transformFlat { + when { + it !is IrProperty -> null + it.backingField?.hasAnnotation(KonanFqNames.volatile) != true -> null + else -> { + val field = it.backingField!! + if (field.type.binaryTypeIsReference() && context.memoryModel != MemoryModel.EXPERIMENTAL) { + it.annotations = it.annotations.filterNot { it.symbol.owner.parentAsClass.hasEqualFqName(KonanFqNames.volatile) } + null + } else { + listOfNotNull(it, + compareAndSetFunction(field), + compareAndSwapFunction(field), + getAndSetFunction(field), + if (field.isInteger()) getAndAddFunction(field) else null + ) + } + } + } + } + declaration.transformChildrenVoid() + return declaration + } + + private fun unsupported(message: String) = builder.irCall(context.ir.symbols.throwIllegalArgumentExceptionWithMessage).apply { + putValueArgument(0, builder.irString(message)) + } + + private val intrinsicMap = mapOf( + IntrinsicType.COMPARE_AND_SET_FIELD to ::compareAndSetFunction, + IntrinsicType.COMPARE_AND_SWAP_FIELD to ::compareAndSwapFunction, + IntrinsicType.GET_AND_SET_FIELD to ::getAndSetFunction, + IntrinsicType.GET_AND_ADD_FIELD to ::getAndAddFunction, + ) + + override fun visitCall(expression: IrCall): IrExpression { + expression.transformChildrenVoid(this) + val intrinsicType = tryGetIntrinsicType(expression).takeIf { it in intrinsicMap } ?: return expression + builder.at(expression) + val reference = expression.getValueArgument(0) as? IrPropertyReference + ?: return unsupported("Only compile-time known IrProperties supported for $intrinsicType") + val property = reference.symbol.owner + val backingField = property.backingField + if (backingField?.type?.binaryTypeIsReference() == true && context.memoryModel != MemoryModel.EXPERIMENTAL) { + return unsupported("Only primitives are supported for $intrinsicType with legacy memory model") + } + if (backingField?.hasAnnotation(KonanFqNames.volatile) != true) { + return unsupported("Only volatile properties are supported for $intrinsicType") + } + val function = intrinsicMap[intrinsicType]!!(backingField) + return builder.irCall(function).apply { + dispatchReceiver = expression.extensionReceiver + putValueArgument(0, expression.getValueArgument(1)) + if (intrinsicType == IntrinsicType.COMPARE_AND_SET_FIELD || intrinsicType == IntrinsicType.COMPARE_AND_SWAP_FIELD) { + putValueArgument(1, expression.getValueArgument(2)) + } + } + } + }) + } +} \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt index d818141106c..b5b0c59ee35 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt @@ -27,6 +27,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.resolve.DescriptorUtils import org.jetbrains.kotlin.util.OperatorNameConventions +import org.jetbrains.kotlin.backend.konan.llvm.* internal class ExternalModulesDFG(val allTypes: List, val publicTypes: Map, @@ -258,7 +259,7 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag context.logMultiple { +"SYMBOL TABLE:" - symbolTable.classMap.forEach { irClass, type -> + symbolTable.classMap.forEach { (irClass, type) -> +" DESCRIPTOR: ${irClass.descriptor}" +" TYPE: $type" if (type !is DataFlowIR.Type.Declared) @@ -309,39 +310,61 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag expressions += expression to currentLoop } - if (expression is IrCall && expression.symbol == initInstanceSymbol) { - // Skip the constructor call as initInstance is handled specially later. - val thiz = expression.getValueArgument(0)!! - val constructorCall = expression.getValueArgument(1)!! - thiz.acceptVoid(this) - constructorCall.acceptChildrenVoid(this) - return + if (expression is IrCall) { + if (expression.symbol == initInstanceSymbol) { + // Skip the constructor call as initInstance is handled specially later. + val thiz = expression.getValueArgument(0)!! + val constructorCall = expression.getValueArgument(1)!! + thiz.acceptVoid(this) + constructorCall.acceptChildrenVoid(this) + return + } + if (expression.symbol == executeImplSymbol) { + // Producer and job of executeImpl are called externally, we need to reflect this somehow. + val producerInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset, + executeImplProducerInvoke.returnType, + executeImplProducerInvoke.symbol, + executeImplProducerInvoke.symbol.owner.typeParameters.size, + executeImplProducerInvoke.symbol.owner.valueParameters.size, + STATEMENT_ORIGIN_PRODUCER_INVOCATION) + producerInvocation.dispatchReceiver = expression.getValueArgument(2) + + expressions += producerInvocation to currentLoop + + val jobFunctionReference = expression.getValueArgument(3) as? IrFunctionReference + ?: error("A function reference expected") + val jobInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset, + jobFunctionReference.symbol.owner.returnType, + jobFunctionReference.symbol as IrSimpleFunctionSymbol, + jobFunctionReference.symbol.owner.typeParameters.size, + jobFunctionReference.symbol.owner.valueParameters.size, + STATEMENT_ORIGIN_JOB_INVOCATION) + jobInvocation.putValueArgument(0, producerInvocation) + + expressions += jobInvocation to currentLoop + } + val intrinsicType = tryGetIntrinsicType(expression) + if (intrinsicType == IntrinsicType.COMPARE_AND_SET || intrinsicType == IntrinsicType.COMPARE_AND_SWAP) { + expressions += IrSetFieldImpl( + expression.startOffset, expression.endOffset, + context.mapping.functionToVolatileField[expression.symbol.owner]!!.symbol, + expression.dispatchReceiver, + expression.getValueArgument(1)!!, + context.irBuiltIns.unitType + ) to currentLoop + } + if (intrinsicType == IntrinsicType.GET_AND_SET) { + expressions += IrSetFieldImpl( + expression.startOffset, expression.endOffset, + context.mapping.functionToVolatileField[expression.symbol.owner]!!.symbol, + expression.dispatchReceiver, + expression.getValueArgument(0)!!, + context.irBuiltIns.unitType + ) to currentLoop + } } - if (expression is IrCall && expression.symbol == executeImplSymbol) { - // Producer and job of executeImpl are called externally, we need to reflect this somehow. - val producerInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset, - executeImplProducerInvoke.returnType, - executeImplProducerInvoke.symbol, - executeImplProducerInvoke.symbol.owner.typeParameters.size, - executeImplProducerInvoke.symbol.owner.valueParameters.size, - STATEMENT_ORIGIN_PRODUCER_INVOCATION) - producerInvocation.dispatchReceiver = expression.getValueArgument(2) - expressions += producerInvocation to currentLoop - - val jobFunctionReference = expression.getValueArgument(3) as? IrFunctionReference - ?: error("A function reference expected") - val jobInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset, - jobFunctionReference.symbol.owner.returnType, - jobFunctionReference.symbol as IrSimpleFunctionSymbol, - jobFunctionReference.symbol.owner.typeParameters.size, - jobFunctionReference.symbol.owner.valueParameters.size, - STATEMENT_ORIGIN_JOB_INVOCATION) - jobInvocation.putValueArgument(0, producerInvocation) - - expressions += jobInvocation to currentLoop - } // TODO: A little bit hacky but it is the simplest solution. // See ObjC instanceOf code generation for details. diff --git a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp index 4903dbc953a..bb8742fadb1 100644 --- a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp +++ b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp @@ -34,6 +34,10 @@ touchFunction(AllocArrayInstance) touchFunction(InitAndRegisterGlobal) touchFunction(UpdateHeapRef) touchFunction(UpdateStackRef) +touchFunction(UpdateVolatileHeapRef) +touchFunction(CompareAndSwapVolatileHeapRef) +touchFunction(CompareAndSetVolatileHeapRef) +touchFunction(GetAndSetVolatileHeapRef) touchFunction(UpdateReturnRef) touchFunction(ZeroHeapRef) touchFunction(ZeroArrayRefs) diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 7c2cae88a4c..4a4a0e2f597 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3343,6 +3343,20 @@ RUNTIME_NOTHROW void UpdateHeapRefRelaxed(ObjHeader** location, const ObjHeader* updateHeapRef(location, object); } +ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) { + RuntimeFail("Shouldn't be used"); +} +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { + RuntimeFail("Shouldn't be used"); +} +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { + RuntimeFail("Shouldn't be used"); +} +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) { + RuntimeFail("Shouldn't be used"); +} + + RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayStrict(const ArrayHeader* array, int fromIndex, int toIndex, int count) { updateHeapRefsInsideOneArray(array, fromIndex, toIndex, count); diff --git a/kotlin-native/runtime/src/main/cpp/Atomic.cpp b/kotlin-native/runtime/src/main/cpp/Atomic.cpp index f9d02f8b4f9..b02ffbbbfd7 100644 --- a/kotlin-native/runtime/src/main/cpp/Atomic.cpp +++ b/kotlin-native/runtime/src/main/cpp/Atomic.cpp @@ -29,41 +29,6 @@ struct AtomicReferenceLayout { KInt cookie_; }; -template struct AtomicPrimitive { - ObjHeader header; - volatile T value_; -}; - -template inline volatile T* getValueLocation(KRef thiz) { - AtomicPrimitive* atomic = reinterpret_cast*>(thiz); - return &atomic->value_; -} - -template void setImpl(KRef thiz, T value) { - volatile T* location = getValueLocation(thiz); - atomicSet(location, value); -} - -template T getImpl(KRef thiz) { - volatile T* location = getValueLocation(thiz); - return atomicGet(location); -} - -template T addAndGetImpl(KRef thiz, T delta) { - volatile T* location = getValueLocation(thiz); - return atomicAdd(location, delta); -} - -template T compareAndSwapImpl(KRef thiz, T expectedValue, T newValue) { - volatile T* location = getValueLocation(thiz); - return compareAndSwap(location, expectedValue, newValue); -} - -template KBoolean compareAndSetImpl(KRef thiz, T expectedValue, T newValue) { - volatile T* location = getValueLocation(thiz); - return compareAndSet(location, expectedValue, newValue); -} - inline AtomicReferenceLayout* asAtomicReference(KRef thiz) { return reinterpret_cast(thiz); } @@ -72,119 +37,7 @@ inline AtomicReferenceLayout* asAtomicReference(KRef thiz) { extern "C" { -KInt Kotlin_AtomicInt_addAndGet(KRef thiz, KInt delta) { - return addAndGetImpl(thiz, delta); -} -KInt Kotlin_AtomicInt_compareAndSwap(KRef thiz, KInt expectedValue, KInt newValue) { - return compareAndSwapImpl(thiz, expectedValue, newValue); -} - -KBoolean Kotlin_AtomicInt_compareAndSet(KRef thiz, KInt expectedValue, KInt newValue) { - return compareAndSetImpl(thiz, expectedValue, newValue); -} - -void Kotlin_AtomicInt_set(KRef thiz, KInt newValue) { - setImpl(thiz, newValue); -} - -KInt Kotlin_AtomicInt_get(KRef thiz) { - return getImpl(thiz); -} - -#if KONAN_NO_64BIT_ATOMIC -static int lock64 = 0; -#endif - -KLong Kotlin_AtomicLong_addAndGet(KRef thiz, KLong delta) { -#if KONAN_NO_64BIT_ATOMIC - // Potentially huge performance penalty, but correct. - while (compareAndSwap(&lock64, 0, 1) != 0); - volatile KLong* address = getValueLocation(thiz); - KLong old = *address; - KLong newValue = old + delta; - *address = newValue; - compareAndSwap(&lock64, 1, 0); - return newValue; -#else - return addAndGetImpl(thiz, delta); -#endif -} - -KLong Kotlin_AtomicLong_compareAndSwap(KRef thiz, KLong expectedValue, KLong newValue) { -#if KONAN_NO_64BIT_ATOMIC - // Potentially huge performance penalty, but correct. - while (compareAndSwap(&lock64, 0, 1) != 0); - volatile KLong* address = getValueLocation(thiz); - KLong old = *address; - if (old == expectedValue) { - *address = newValue; - } - compareAndSwap(&lock64, 1, 0); - return old; -#else - return compareAndSwapImpl(thiz, expectedValue, newValue); -#endif -} - -KBoolean Kotlin_AtomicLong_compareAndSet(KRef thiz, KLong expectedValue, KLong newValue) { -#if KONAN_NO_64BIT_ATOMIC - // Potentially huge performance penalty, but correct. - KBoolean result = false; - while (compareAndSwap(&lock64, 0, 1) != 0); - volatile KLong* address = getValueLocation(thiz); - KLong old = *address; - if (old == expectedValue) { - result = true; - *address = newValue; - } - compareAndSwap(&lock64, 1, 0); - return result; -#else - return compareAndSetImpl(thiz, expectedValue, newValue); -#endif -} - -void Kotlin_AtomicLong_set(KRef thiz, KLong newValue) { -#if KONAN_NO_64BIT_ATOMIC - // Potentially huge performance penalty, but correct. - while (compareAndSwap(&lock64, 0, 1) != 0); - volatile KLong* address = getValueLocation(thiz); - *address = newValue; - compareAndSwap(&lock64, 1, 0); -#else - setImpl(thiz, newValue); -#endif -} - -KLong Kotlin_AtomicLong_get(KRef thiz) { -#if KONAN_NO_64BIT_ATOMIC - // Potentially huge performance penalty, but correct. - while (compareAndSwap(&lock64, 0, 1) != 0); - volatile KLong* address = getValueLocation(thiz); - KLong value = *address; - compareAndSwap(&lock64, 1, 0); - return value; -#else - return getImpl(thiz); -#endif -} - -KNativePtr Kotlin_AtomicNativePtr_compareAndSwap(KRef thiz, KNativePtr expectedValue, KNativePtr newValue) { - return compareAndSwapImpl(thiz, expectedValue, newValue); -} - -KBoolean Kotlin_AtomicNativePtr_compareAndSet(KRef thiz, KNativePtr expectedValue, KNativePtr newValue) { - return compareAndSetImpl(thiz, expectedValue, newValue); -} - -void Kotlin_AtomicNativePtr_set(KRef thiz, KNativePtr newValue) { - setImpl(thiz, newValue); -} - -KNativePtr Kotlin_AtomicNativePtr_get(KRef thiz) { - return getImpl(thiz); -} void Kotlin_AtomicReference_checkIfFrozen(KRef value) { if (!kotlin::compiler::freezingEnabled()) { diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 72501d1a53c..cb45afce7eb 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -247,6 +247,12 @@ void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW; void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates heap/static data location. void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Updates volatile heap/static data location. +void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) RUNTIME_NOTHROW; +bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) RUNTIME_NOTHROW; +OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) RUNTIME_NOTHROW; + // Updates heap/static data in one array. void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW; // Updates location if it is null, atomically. diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/concurrent/Atomics.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/concurrent/Atomics.kt index 40dc9490d8a..270875b264b 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/concurrent/Atomics.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/concurrent/Atomics.kt @@ -7,6 +7,8 @@ package kotlin.native.concurrent import kotlinx.cinterop.NativePtr import kotlin.native.internal.* +import kotlin.reflect.* +import kotlin.concurrent.* /** * Wrapper around [Int] with atomic synchronized operations. @@ -16,23 +18,15 @@ import kotlin.native.internal.* * So shared frozen objects can have mutable fields of [AtomicInt] type. */ @Frozen -@OptIn(FreezingIsDeprecated::class) -public class AtomicInt(private var value_: Int) { - /** - * The value being held by this class. - */ - public var value: Int - get() = getImpl() - set(new) = setImpl(new) - +@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class) +public class AtomicInt(public @Volatile var value: Int) { /** * Increments the value by [delta] and returns the new value. * * @param delta the value to add * @return the new value */ - @GCUnsafeCall("Kotlin_AtomicInt_addAndGet") - external public fun addAndGet(delta: Int): Int + public fun addAndGet(delta: Int): Int = getAndAddField(AtomicInt::value, delta) + delta /** * Compares value with [expected] and replaces it with [new] value if values matches. @@ -41,8 +35,7 @@ public class AtomicInt(private var value_: Int) { * @param new the new value * @return the old value */ - @GCUnsafeCall("Kotlin_AtomicInt_compareAndSwap") - external public fun compareAndSwap(expected: Int, new: Int): Int + public fun compareAndSwap(expected: Int, new: Int): Int = compareAndSwapField(AtomicInt::value, expected, new) /** * Compares value with [expected] and replaces it with [new] value if values matches. @@ -51,8 +44,7 @@ public class AtomicInt(private var value_: Int) { * @param new the new value * @return true if successful */ - @GCUnsafeCall("Kotlin_AtomicInt_compareAndSet") - external public fun compareAndSet(expected: Int, new: Int): Boolean + public fun compareAndSet(expected: Int, new: Int): Boolean = compareAndSetField(AtomicInt::value, expected, new) /** * Increments value by one. @@ -74,13 +66,6 @@ public class AtomicInt(private var value_: Int) { * @return the string representation */ public override fun toString(): String = value.toString() - - // Implementation details. - @GCUnsafeCall("Kotlin_AtomicInt_set") - private external fun setImpl(new: Int): Unit - - @GCUnsafeCall("Kotlin_AtomicInt_get") - private external fun getImpl(): Int } /** @@ -91,14 +76,8 @@ public class AtomicInt(private var value_: Int) { * So shared frozen objects can have mutable fields of [AtomicLong] type. */ @Frozen -@OptIn(FreezingIsDeprecated::class) -public class AtomicLong(private var value_: Long = 0) { - /** - * The value being held by this class. - */ - public var value: Long - get() = getImpl() - set(new) = setImpl(new) +@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class) +public class AtomicLong(public @Volatile var value: Long = 0) { /** * Increments the value by [delta] and returns the new value. @@ -106,8 +85,7 @@ public class AtomicLong(private var value_: Long = 0) { * @param delta the value to add * @return the new value */ - @GCUnsafeCall("Kotlin_AtomicLong_addAndGet") - external public fun addAndGet(delta: Long): Long + public fun addAndGet(delta: Long): Long = getAndAddField(AtomicLong::value, delta) + delta /** * Increments the value by [delta] and returns the new value. @@ -124,8 +102,7 @@ public class AtomicLong(private var value_: Long = 0) { * @param new the new value * @return the old value */ - @GCUnsafeCall("Kotlin_AtomicLong_compareAndSwap") - external public fun compareAndSwap(expected: Long, new: Long): Long + public fun compareAndSwap(expected: Long, new: Long): Long = compareAndSwapField(AtomicLong::value, expected, new) /** * Compares value with [expected] and replaces it with [new] value if values matches. @@ -134,8 +111,7 @@ public class AtomicLong(private var value_: Long = 0) { * @param new the new value * @return true if successful, false if state is unchanged */ - @GCUnsafeCall("Kotlin_AtomicLong_compareAndSet") - external public fun compareAndSet(expected: Long, new: Long): Boolean + public fun compareAndSet(expected: Long, new: Long): Boolean = compareAndSetField(AtomicLong::value, expected, new) /** * Increments value by one. @@ -157,13 +133,6 @@ public class AtomicLong(private var value_: Long = 0) { * @return the string representation of this object */ public override fun toString(): String = value.toString() - - // Implementation details. - @GCUnsafeCall("Kotlin_AtomicLong_set") - private external fun setImpl(new: Long): Unit - - @GCUnsafeCall("Kotlin_AtomicLong_get") - private external fun getImpl(): Long } /** @@ -174,14 +143,8 @@ public class AtomicLong(private var value_: Long = 0) { * So shared frozen objects can have mutable fields of [AtomicNativePtr] type. */ @Frozen -@OptIn(FreezingIsDeprecated::class) -public class AtomicNativePtr(private var value_: NativePtr) { - /** - * The value being held by this class. - */ - public var value: NativePtr - get() = getImpl() - set(new) = setImpl(new) +@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class) +public class AtomicNativePtr(public @Volatile var value: NativePtr) { /** * Compares value with [expected] and replaces it with [new] value if values matches. @@ -190,8 +153,8 @@ public class AtomicNativePtr(private var value_: NativePtr) { * @param new the new value * @return the old value */ - @GCUnsafeCall("Kotlin_AtomicNativePtr_compareAndSwap") - external public fun compareAndSwap(expected: NativePtr, new: NativePtr): NativePtr + public fun compareAndSwap(expected: NativePtr, new: NativePtr): NativePtr = + compareAndSwapField(AtomicNativePtr::value, expected, new) /** * Compares value with [expected] and replaces it with [new] value if values matches. @@ -200,8 +163,8 @@ public class AtomicNativePtr(private var value_: NativePtr) { * @param new the new value * @return true if successful */ - @GCUnsafeCall("Kotlin_AtomicNativePtr_compareAndSet") - external public fun compareAndSet(expected: NativePtr, new: NativePtr): Boolean + public fun compareAndSet(expected: NativePtr, new: NativePtr): Boolean = + compareAndSetField(AtomicNativePtr::value, expected, new) /** * Returns the string representation of this object. @@ -209,13 +172,6 @@ public class AtomicNativePtr(private var value_: NativePtr) { * @return string representation of this object */ public override fun toString(): String = value.toString() - - // Implementation details. - @GCUnsafeCall("Kotlin_AtomicNativePtr_set") - private external fun setImpl(new: NativePtr): Unit - - @GCUnsafeCall("Kotlin_AtomicNativePtr_get") - private external fun getImpl(): NativePtr } @@ -431,3 +387,134 @@ public class FreezableAtomicReference(private var value_: T) { @GCUnsafeCall("Kotlin_AtomicReference_compareAndSet") private external fun compareAndSetImpl(expected: Any?, new: Any?): Boolean } + + +/** + * Compares the value of the field referenced by [fieldRef] from [this] object to [expectedValue], and if they are equal, + * atomically replaces it with [newValue]. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * Comparison is done by reference or value depending on field representation. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Returns true if the actual field value matched [expectedValue] + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.COMPARE_AND_SET_FIELD) +internal external fun T.compareAndSetField(filedRef: KMutableProperty1, expectedValue: S, newValue: S): Boolean + +/** + * Compares the value of the field referenced by [fieldRef] from [this] object to [expectedValue], and if they are equal, + * atomically replaces it with [newValue]. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * Comparison is done by reference or value depending on field representation. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Returns true if the actual field value before operation. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.COMPARE_AND_SWAP_FIELD) +internal external fun T.compareAndSwapField(filedRef: KMutableProperty1, expectedValue: S, newValue: S): S + +/** + * Atomically sets value of the field referenced by [fieldRef] from [this] object to [newValue] and returns old field value. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.GET_AND_SET_FIELD) +internal external fun T.getAndSetField(filedRef: KMutableProperty1, newValue: S): S + + +/** + * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD) +internal external fun T.getAndAddField(filedRef: KMutableProperty1, delta: Short): Short + +/** + * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD) +internal external fun T.getAndAddField(filedRef: KMutableProperty1, newValue: Int): Int + +/** + * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD) +internal external fun T.getAndAddField(filedRef: KMutableProperty1, newValue: Long): Long + +/** + * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value. + * + * For now, it can be used only within the same file, where class [T] is defined. + * Check https://youtrack.jetbrains.com/issue/KT-55426 for details. + * + * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException] + * would be thrown. + * + * If property referenced by [fieldRef] has nontrivial setter it will not be called. + * + * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown. + */ +@PublishedApi +@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD) +internal external fun T.getAndAddField(filedRef: KMutableProperty1, newValue: Byte): Byte diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt index ad5913ad8e5..0b07f36129b 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt @@ -80,5 +80,15 @@ class IntrinsicType { // Worker const val WORKER_EXECUTE = "WORKER_EXECUTE" + + // Atomic + const val COMPARE_AND_SET_FIELD = "COMPARE_AND_SET_FIELD" + const val COMPARE_AND_SWAP_FIELD = "COMPARE_AND_SWAP_FIELD" + const val GET_AND_SET_FIELD = "GET_AND_SET_FIELD" + const val GET_AND_ADD_FIELD = "GET_AND_ADD_FIELD" + const val COMPARE_AND_SET = "COMPARE_AND_SET" + const val COMPARE_AND_SWAP = "COMPARE_AND_SWAP" + const val GET_AND_SET = "GET_AND_SET" + const val GET_AND_ADD = "GET_AND_ADD" } } \ No newline at end of file diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 91096c37db5..26663639fbc 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -188,6 +188,22 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location mm::SetHeapRef(location, const_cast(object)); } +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) { + mm::SetHeapRefAtomicSeqCst(location, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { + RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { + return mm::CompareAndSetHeapRef(location, expectedValue, newValue); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) { + RETURN_RESULT_OF(mm::GetAndSetHeapRef, location, newValue); +} + extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { if (object == nullptr) return; ObjHeader* result = nullptr; // No need to store this value in a rootset. diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index b72df3f89fa..7444497ff62 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -34,6 +34,12 @@ ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) __atomic_store_n(location, value, __ATOMIC_RELEASE); } +ALWAYS_INLINE void mm::SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept { + AssertThreadState(ThreadState::kRunnable); + __atomic_store_n(location, value, __ATOMIC_SEQ_CST); +} + + ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept { AssertThreadState(ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. @@ -45,14 +51,24 @@ ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHea AssertThreadState(ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. ObjHeader* actual = expected; - // TODO: Do we need this strong memory model? Do we need to use strong CAS? - // This intrinsic modifies `actual` non-atomically. __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); - // On success, we already have old value (== `expected`) in `actual`. - // On failure, we have the old value written into `actual`. RETURN_OBJ(actual); } +ALWAYS_INLINE bool mm::CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept { + AssertThreadState(ThreadState::kRunnable); + // TODO: Make this work with GCs that can stop thread at any point. + ObjHeader* actual = expected; + return __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept { + AssertThreadState(ThreadState::kRunnable);; + auto *actual = __atomic_exchange_n(location, value, __ATOMIC_SEQ_CST); + RETURN_OBJ(actual); +} + + #pragma clang diagnostic pop OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp index 9e06fa89b16..a4075be7aac 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -24,8 +24,11 @@ class ThreadData; void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept; void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept; void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept; +void SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept; OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept; OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; +bool CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; +OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept; OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;