[K/N] Support of @Volatile annotation

^KT-54944

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -50,6 +50,10 @@ import kotlin.LazyThreadSafetyMode.PUBLICATION
 
 internal class NativeMapping : DefaultMapping() {
     data class BridgeKey(val target: IrSimpleFunction, val bridgeDirections: BridgeDirections)
+    enum class AtomicFunctionType {
+        COMPARE_AND_SWAP, COMPARE_AND_SET, GET_AND_SET, GET_AND_ADD;
+    }
+    data class AtomicFunctionKey(val field: IrField, val type: AtomicFunctionType)
 
     val outerThisFields = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrClass, IrField>()
     val enumValueGetters = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrClass, IrFunction>()
@@ -62,6 +66,8 @@ internal class NativeMapping : DefaultMapping() {
     val boxFunctions = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrClass, IrSimpleFunction>()
     val unboxFunctions = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrClass, IrSimpleFunction>()
     val loweredInlineClassConstructors = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrConstructor, IrSimpleFunction>()
+    val volatileFieldToAtomicFunction = mutableMapOf<AtomicFunctionKey, IrSimpleFunction>()
+    val functionToVolatileField = DefaultDelegateFactory.newDeclarationToDeclarationMapping<IrSimpleFunction, IrField>()
 }
 
 internal class Context(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanFqNames.kt

@@ -25,6 +25,7 @@ object KonanFqNames {
     val cancellationException = FqName("kotlin.coroutines.cancellation.CancellationException")
     val threadLocal = FqName("kotlin.native.concurrent.ThreadLocal")
     val sharedImmutable = FqName("kotlin.native.concurrent.SharedImmutable")
+    val volatile = FqName("kotlin.concurrent.Volatile")
     val frozen = FqName("kotlin.native.internal.Frozen")
     val frozenLegacyMM = FqName("kotlin.native.internal.FrozenLegacyMM")
     val leakDetectorCandidate = FqName("kotlin.native.internal.LeakDetectorCandidate")

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanLoweringPhases.kt

@@ -274,6 +274,13 @@ internal val tailrecPhase = makeKonanFileLoweringPhase(
         prerequisite = setOf(localFunctionsPhase)
 )
 
+internal val volatilePhase = makeKonanFileLoweringPhase(
+        ::VolatileFieldsLowering,
+        name = "VolatileFields",
+        description = "Volatile fields processing",
+        prerequisite = setOf(localFunctionsPhase)
+)
+
 internal val defaultParameterExtentPhase = makeKonanFileOpPhase(
         { context, irFile ->
             KonanDefaultArgumentStubGenerator(context).lower(irFile)
@@ -335,7 +342,8 @@ internal val testProcessorPhase = makeKonanFileOpPhase(
 internal val delegationPhase = makeKonanFileLoweringPhase(
         { PropertyDelegationLowering(it.generationState) },
         name = "Delegation",
-        description = "Delegation lowering"
+        description = "Delegation lowering",
+        prerequisite = setOf(volatilePhase)
 )
 
 internal val functionReferencePhase = makeKonanFileLoweringPhase(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt

@@ -165,6 +165,7 @@ internal val allLoweringsPhase = SameTypeNamedCompilerPhase(
                         enumConstructorsPhase,
                         initializersPhase,
                         localFunctionsPhase,
+                        volatilePhase,
                         tailrecPhase,
                         defaultParameterExtentPhase,
                         innerClassPhase,

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -99,6 +99,7 @@ internal abstract class KonanSymbols(
     val symbolName = topLevelClass(RuntimeNames.symbolNameAnnotation)
     val filterExceptions = topLevelClass(RuntimeNames.filterExceptions)
     val exportForCppRuntime = topLevelClass(RuntimeNames.exportForCppRuntime)
+    val typedIntrinsic = topLevelClass(RuntimeNames.typedIntrinsicAnnotation)
 
     val objCMethodImp = symbolTable.referenceClass(descriptorsLookup.interopBuiltIns.objCMethodImp)
 
@@ -415,6 +416,8 @@ internal abstract class KonanSymbols(
 
     val sharedImmutable = topLevelClass(KonanFqNames.sharedImmutable)
 
+    val volatile = topLevelClass(KonanFqNames.volatile)
+
     val eagerInitialization = topLevelClass(KonanFqNames.eagerInitialization)
 
     private fun topLevelClass(fqName: FqName): IrClassSymbol = irBuiltIns.findClass(fqName.shortName(), fqName.parent())!!

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -560,21 +560,22 @@ internal abstract class FunctionGenerationContext(
 
     fun param(index: Int): LLVMValueRef = LLVMGetParam(this.function, index)!!
 
-    fun load(address: LLVMValueRef, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef {
+    fun load(address: LLVMValueRef, name: String = "",
+             memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null
+    ): LLVMValueRef {
         val value = LLVMBuildLoad(builder, address, name)!!
-        if (memoryOrder != null) {
-            LLVMSetOrdering(value, memoryOrder)
-        }
+        memoryOrder?.let { LLVMSetOrdering(value, it) }
+        alignment?.let { LLVMSetAlignment(value, it) }
         // Use loadSlot() API for that.
         assert(!isObjectRef(value))
         return value
     }
 
-    fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "", memoryOrder: LLVMAtomicOrdering? = null): LLVMValueRef {
+    fun loadSlot(address: LLVMValueRef, isVar: Boolean, resultSlot: LLVMValueRef? = null, name: String = "",
+                 memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null): LLVMValueRef {
         val value = LLVMBuildLoad(builder, address, name)!!
-        if (memoryOrder != null) {
-            LLVMSetOrdering(value, memoryOrder)
-        }
+        memoryOrder?.let { LLVMSetOrdering(value, it) }
+        alignment?.let { LLVMSetAlignment(value, it) }
         if (isObjectRef(value) && isVar) {
             val slot = resultSlot ?: alloca(LLVMTypeOf(value), variableLocation = null)
             storeStackRef(value, slot)
@@ -582,8 +583,10 @@ internal abstract class FunctionGenerationContext(
         return value
     }
 
-    fun store(value: LLVMValueRef, ptr: LLVMValueRef) {
-        LLVMBuildStore(builder, value, ptr)
+    fun store(value: LLVMValueRef, ptr: LLVMValueRef, memoryOrder: LLVMAtomicOrdering? = null, alignment: Int? = null) {
+        val store = LLVMBuildStore(builder, value, ptr)
+        memoryOrder?.let { LLVMSetOrdering(store, it) }
+        alignment?.let { LLVMSetAlignment(store, it) }
     }
 
     fun storeHeapRef(value: LLVMValueRef, ptr: LLVMValueRef) {
@@ -594,12 +597,12 @@ internal abstract class FunctionGenerationContext(
         updateRef(value, ptr, onStack = true)
     }
 
-    fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean) = if (isObjectRef(value)) {
-            if (onStack) storeStackRef(value, ptr) else storeHeapRef(value, ptr)
-            null
-        } else {
-            LLVMBuildStore(builder, value, ptr)
+    fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean, isVolatile: Boolean = false, alignment: Int? = null) {
+        when {
+            isObjectRef(value) -> updateRef(value, ptr, onStack, isVolatile, alignment)
+            else -> store(value, ptr, if (isVolatile) LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent else null, alignment)
         }
+    }
 
     fun freeze(value: LLVMValueRef, exceptionHandler: ExceptionHandler) {
         if (isObjectRef(value))
@@ -618,14 +621,21 @@ internal abstract class FunctionGenerationContext(
             call(llvm.updateReturnRefFunction, listOf(address, value))
     }
 
-    private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean) {
+    private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean,
+                          isVolatile: Boolean = false, alignment: Int? = null) {
+        require(alignment == null || alignment == runtime.pointerAlignment)
         if (onStack) {
+            require(!isVolatile) { "Stack ref update can't be volatile"}
             if (context.memoryModel == MemoryModel.STRICT)
                 store(value, address)
             else
                 call(llvm.updateStackRefFunction, listOf(address, value))
         } else {
-            call(llvm.updateHeapRefFunction, listOf(address, value))
+            if (isVolatile && context.memoryModel == MemoryModel.EXPERIMENTAL) {
+                call(llvm.UpdateVolatileHeapRef, listOf(address, value))
+            } else {
+                call(llvm.updateHeapRefFunction, listOf(address, value))
+            }
         }
     }
 
@@ -1349,7 +1359,7 @@ internal abstract class FunctionGenerationContext(
             }
             addPhiIncoming(slotsPhi!!, prologueBb to slots)
             memScoped {
-                slotToVariableLocation.forEach { slot, variable ->
+                slotToVariableLocation.forEach { (slot, variable) ->
                     val expr = longArrayOf(DwarfOp.DW_OP_plus_uconst.value,
                             runtime.pointerSize * slot.toLong()).toCValues()
                     DIInsertDeclaration(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -457,6 +457,12 @@ internal class Llvm(private val generationState: NativeGenerationState, val modu
     val Kotlin_processFieldInMark by lazyRtFunction
     val Kotlin_processEmptyObjectInMark by lazyRtFunction
 
+    val UpdateVolatileHeapRef by lazyRtFunction
+    val CompareAndSetVolatileHeapRef by lazyRtFunction
+    val CompareAndSwapVolatileHeapRef by lazyRtFunction
+    val GetAndSetVolatileHeapRef by lazyRtFunction
+
+
     val tlsMode by lazy {
         when (target) {
             KonanTarget.WASM32,

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt

@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.backend.konan.llvm.objc.genObjCSelector
 import org.jetbrains.kotlin.backend.konan.reportCompilationError
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrField
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.symbols.IrConstructorSymbol
@@ -90,7 +91,16 @@ internal enum class IntrinsicType {
     INTEROP_FUNPTR_INVOKE,
     INTEROP_MEMORY_COPY,
     // Worker
-    WORKER_EXECUTE
+    WORKER_EXECUTE,
+    // Atomics
+    COMPARE_AND_SET_FIELD,
+    COMPARE_AND_SWAP_FIELD,
+    GET_AND_SET_FIELD,
+    GET_AND_ADD_FIELD,
+    COMPARE_AND_SET,
+    COMPARE_AND_SWAP,
+    GET_AND_SET,
+    GET_AND_ADD,
 }
 
 internal enum class ConstantConstructorIntrinsicType {
@@ -117,6 +127,8 @@ internal interface IntrinsicGeneratorEnvironment {
     fun evaluateExplicitArgs(expression: IrFunctionAccessExpression): List<LLVMValueRef>
 
     fun evaluateExpression(value: IrExpression, resultSlot: LLVMValueRef?): LLVMValueRef
+
+    fun getObjectFieldPointer(thisRef: LLVMValueRef, field: IrField): LLVMValueRef
 }
 
 internal fun tryGetIntrinsicType(callSite: IrFunctionAccessExpression): IntrinsicType? =
@@ -242,6 +254,10 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                 IntrinsicType.INTEROP_MEMORY_COPY -> emitMemoryCopy(callSite, args)
                 IntrinsicType.IS_EXPERIMENTAL_MM -> emitIsExperimentalMM()
                 IntrinsicType.THE_UNIT_INSTANCE -> theUnitInstanceRef.llvm
+                IntrinsicType.COMPARE_AND_SET -> emitCompareAndSet(callSite, args)
+                IntrinsicType.COMPARE_AND_SWAP -> emitCompareAndSwap(callSite, args, resultSlot)
+                IntrinsicType.GET_AND_SET -> emitGetAndSet(callSite, args, resultSlot)
+                IntrinsicType.GET_AND_ADD -> emitGetAndAdd(callSite, args)
                 IntrinsicType.GET_CONTINUATION,
                 IntrinsicType.RETURN_IF_SUSPENDED,
                 IntrinsicType.INTEROP_BITS_TO_FLOAT,
@@ -253,7 +269,11 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                 IntrinsicType.INTEROP_CONVERT,
                 IntrinsicType.ENUM_VALUES,
                 IntrinsicType.ENUM_VALUE_OF,
-                IntrinsicType.WORKER_EXECUTE ->
+                IntrinsicType.WORKER_EXECUTE,
+                IntrinsicType.COMPARE_AND_SET_FIELD,
+                IntrinsicType.COMPARE_AND_SWAP_FIELD,
+                IntrinsicType.GET_AND_SET_FIELD,
+                IntrinsicType.GET_AND_ADD_FIELD ->
                     reportNonLoweredIntrinsic(intrinsicType)
                 IntrinsicType.INIT_INSTANCE,
                 IntrinsicType.OBJC_INIT_BY,
@@ -292,6 +312,65 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
     private fun FunctionGenerationContext.emitIsExperimentalMM(): LLVMValueRef =
             llvm.int1(context.memoryModel == MemoryModel.EXPERIMENTAL)
 
+    // cmpxcgh llvm instruction return pair. idnex is index of required element of this pair
+    enum class CmpExchangeMode(val index:Int) {
+        SWAP(0),
+        SET(1)
+    }
+
+    private fun FunctionGenerationContext.emitCmpExchange(callSite: IrCall, args: List<LLVMValueRef>, mode: CmpExchangeMode, resultSlot: LLVMValueRef?): LLVMValueRef {
+        val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!!
+        require(args.size == 3)
+        val address = environment.getObjectFieldPointer(args[0], field)
+        return if (isObjectRef(args[1])) {
+            require(context.memoryModel == MemoryModel.EXPERIMENTAL)
+            when (mode) {
+                CmpExchangeMode.SET -> call(llvm.CompareAndSetVolatileHeapRef, listOf(address, args[1], args[2]))
+                CmpExchangeMode.SWAP -> call(llvm.CompareAndSwapVolatileHeapRef, listOf(address, args[1], args[2]),
+                        environment.calculateLifetime(callSite), resultSlot = resultSlot)
+            }
+        } else {
+            val cmp = LLVMBuildAtomicCmpXchg(builder, address, args[1], args[2],
+                    LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
+                    LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
+                    SingleThread = 0
+            )!!
+
+            LLVMBuildExtractValue(builder, cmp, mode.index, "")!!
+        }
+    }
+
+    private fun FunctionGenerationContext.emitAtomicRMW(callSite: IrCall, args: List<LLVMValueRef>, op: LLVMAtomicRMWBinOp, resultSlot: LLVMValueRef?): LLVMValueRef {
+        val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!!
+        require(args.size == 2)
+        val address = environment.getObjectFieldPointer(args[0], field)
+        return if (isObjectRef(args[1])) {
+            require(op == LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg)
+            require(context.memoryModel == MemoryModel.EXPERIMENTAL)
+            call(llvm.GetAndSetVolatileHeapRef, listOf(address, args[1]),
+                    environment.calculateLifetime(callSite), resultSlot = resultSlot)
+        } else {
+            LLVMBuildAtomicRMW(builder, op, address, args[1],
+                    LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
+                    singleThread = 0
+            )!!
+        }
+    }
+
+    private fun FunctionGenerationContext.emitCompareAndSet(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
+        return emitCmpExchange(callSite, args, CmpExchangeMode.SET, null)
+    }
+    private fun FunctionGenerationContext.emitCompareAndSwap(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
+        return emitCmpExchange(callSite, args, CmpExchangeMode.SWAP, resultSlot)
+    }
+    private fun FunctionGenerationContext.emitGetAndSet(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
+        return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg, resultSlot)
+    }
+    private fun FunctionGenerationContext.emitGetAndAdd(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
+        return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpAdd, null)
+    }
+
+
     private fun FunctionGenerationContext.emitGetNativeNullPtr(): LLVMValueRef =
             llvm.kNullInt8Ptr
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -228,6 +228,9 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
 
         override fun evaluateExpression(value: IrExpression, resultSlot: LLVMValueRef?): LLVMValueRef =
                 this@CodeGeneratorVisitor.evaluateExpression(value, resultSlot)
+
+        override fun getObjectFieldPointer(thisRef: LLVMValueRef, field: IrField): LLVMValueRef =
+                this@CodeGeneratorVisitor.fieldPtrOfClass(thisRef, field)
     }
 
     private val intrinsicGenerator = IntrinsicGenerator(intrinsicGeneratorEnvironment)
@@ -1660,35 +1663,46 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
 
     private fun evaluateGetField(value: IrGetField, resultSlot: LLVMValueRef?): LLVMValueRef {
         context.log { "evaluateGetField               : ${ir2string(value)}" }
-        return if (!value.symbol.owner.isStatic) {
-            val thisPtr = evaluateExpression(value.receiver!!)
-            functionGenerationContext.loadSlot(
-                    fieldPtrOfClass(thisPtr, value.symbol.owner), !value.symbol.owner.isFinal, resultSlot)
-        } else {
-            assert(value.receiver == null)
-            if (value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true) {
-                evaluateConst(value.symbol.owner.initializer?.expression as IrConst<*>).llvm
-            } else {
+        val alignment = when {
+            value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8
+            else -> null
+        }
+        val order = when {
+            value.symbol.owner.hasAnnotation(KonanFqNames.volatile) ->
+                LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent
+            else -> null
+        }
+
+        val fieldAddress = when {
+            !value.symbol.owner.isStatic -> {
+                fieldPtrOfClass(evaluateExpression(value.receiver!!), value.symbol.owner)
+            }
+            value.symbol.owner.correspondingPropertySymbol?.owner?.isConst == true -> {
+                // TODO: probably can be removed, as they are inlined.
+                return evaluateConst(value.symbol.owner.initializer?.expression as IrConst<*>).llvm
+            }
+            else -> {
                 if (context.config.threadsAreAllowed && value.symbol.owner.isGlobalNonPrimitive(context)) {
                     functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler)
                 }
-                val ptr = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress(
-                        functionGenerationContext
-                )
-                functionGenerationContext.loadSlot(ptr, !value.symbol.owner.isFinal, resultSlot)
+                generationState.llvmDeclarations
+                        .forStaticField(value.symbol.owner)
+                        .storageAddressAccess
+                        .getAddress(functionGenerationContext)
             }
-        }.also {
-            if (value.type.classifierOrNull?.isClassWithFqName(vectorType) == true)
-                LLVMSetAlignment(it, 8)
-
         }
+        return functionGenerationContext.loadSlot(
+                fieldAddress, !value.symbol.owner.isFinal, resultSlot,
+                memoryOrder = order,
+                alignment = alignment
+        )
     }
 
     //-------------------------------------------------------------------------//
-    private fun needMutationCheck(irClass: IrClass): Boolean {
+    private fun needMutationCheck(irField: IrField): Boolean {
         // For now we omit mutation checks on immutable types, as this allows initialization in constructor
         // and it is assumed that API doesn't allow to change them.
-        return context.config.freezing.enableFreezeChecks && !irClass.isFrozen(context)
+        return context.config.freezing.enableFreezeChecks && !irField.parentAsClass.isFrozen(context) && !irField.hasAnnotation(KonanFqNames.volatile)
     }
 
     private fun needLifetimeConstraintsCheck(valueToAssign: LLVMValueRef, irClass: IrClass): Boolean {
@@ -1724,13 +1738,14 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
         }
 
         val valueToAssign = evaluateExpression(value.value)
-        val store = if (!value.symbol.owner.isStatic) {
+        val address: LLVMValueRef
+        if (!value.symbol.owner.isStatic) {
             val thisPtr = evaluateExpression(value.receiver!!)
             assert(thisPtr.type == codegen.kObjHeaderPtr) {
                 LLVMPrintTypeToString(thisPtr.type)?.toKString().toString()
             }
             val parentAsClass = value.symbol.owner.parentAsClass
-            if (needMutationCheck(parentAsClass)) {
+            if (needMutationCheck(value.symbol.owner)) {
                 functionGenerationContext.call(llvm.mutationCheck,
                         listOf(functionGenerationContext.bitcast(codegen.kObjHeaderPtr, thisPtr)),
                         Lifetime.IRRELEVANT, currentCodeContext.exceptionHandler)
@@ -1738,21 +1753,26 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
             if (needLifetimeConstraintsCheck(valueToAssign, parentAsClass)) {
                 functionGenerationContext.call(llvm.checkLifetimesConstraint, listOf(thisPtr, valueToAssign))
             }
-            functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner), false)
+            address = fieldPtrOfClass(thisPtr, value.symbol.owner)
         } else {
             assert(value.receiver == null)
-            val globalAddress = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress(
-                    functionGenerationContext
-            )
             if (context.config.threadsAreAllowed && value.symbol.owner.storageKind(context) == FieldStorageKind.GLOBAL)
                 functionGenerationContext.checkGlobalsAccessible(currentCodeContext.exceptionHandler)
             if (value.symbol.owner.shouldBeFrozen(context) && value.origin != ObjectClassLowering.IrStatementOriginFieldPreInit)
                 functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler)
-            functionGenerationContext.storeAny(valueToAssign, globalAddress, false)
+            address = generationState.llvmDeclarations.forStaticField(value.symbol.owner).storageAddressAccess.getAddress(
+                    functionGenerationContext
+            )
         }
-        if (store != null && value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true) {
-            LLVMSetAlignment(store, 8)
+        val alignment = when {
+            value.value.type.classifierOrNull?.isClassWithFqName(vectorType) == true -> 8
+            else -> null
         }
+        functionGenerationContext.storeAny(
+                valueToAssign, address, false,
+                isVolatile = value.symbol.owner.hasAnnotation(KonanFqNames.volatile),
+                alignment = alignment,
+        )
 
         assert (value.type.isUnit())
         return codegen.theUnitInstanceRef.llvm
@@ -2379,8 +2399,7 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
         val bbExit = basicBlock("label_continue", null)
         moveBlockAfterEntry(bbExit)
         moveBlockAfterEntry(bbInit)
-        val state = load(statePtr)
-        LLVMSetOrdering(state, LLVMAtomicOrdering.LLVMAtomicOrderingAcquire)
+        val state = load(statePtr, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingAcquire)
         condBr(icmpEq(state, llvm.int32(FILE_INITIALIZED)), bbExit, bbInit)
         positionAtEnd(bbInit)
         call(llvm.callInitGlobalPossiblyLock, listOf(statePtr, initializerPtr),

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/VolatileFieldsLowering.kt

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.backend.konan.lower
+
+import org.jetbrains.kotlin.backend.common.FileLoweringPass
+import org.jetbrains.kotlin.backend.common.ir.addDispatchReceiver
+import org.jetbrains.kotlin.backend.common.lower.*
+import org.jetbrains.kotlin.backend.konan.*
+import org.jetbrains.kotlin.backend.konan.Context
+import org.jetbrains.kotlin.backend.konan.NativeMapping
+import org.jetbrains.kotlin.backend.konan.ir.buildSimpleAnnotation
+import org.jetbrains.kotlin.backend.konan.llvm.IntrinsicType
+import org.jetbrains.kotlin.backend.konan.llvm.tryGetIntrinsicType
+import org.jetbrains.kotlin.ir.IrStatement
+import org.jetbrains.kotlin.ir.builders.declarations.addValueParameter
+import org.jetbrains.kotlin.ir.builders.declarations.buildFun
+import org.jetbrains.kotlin.ir.builders.irString
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.expressions.*
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.util.*
+import org.jetbrains.kotlin.ir.visitors.*
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.util.capitalizeDecapitalize.*
+
+object IR_DECLARATION_ORIGIN_VOLATILE : IrDeclarationOriginImpl("VOLATILE")
+
+internal class VolatileFieldsLowering(val context: Context) : FileLoweringPass {
+    private fun buildIntrinsicFunction(irField: IrField, intrinsicType: IntrinsicType, builder: IrSimpleFunction.() -> Unit) = context.irFactory.buildFun {
+        isExternal = true
+        origin = IR_DECLARATION_ORIGIN_VOLATILE
+        name = Name.special("<${intrinsicType.name.decapitalizeSmart()}-${irField.name}>")
+        startOffset = irField.startOffset
+        endOffset = irField.endOffset
+    }.apply {
+        val parentClass = irField.parents.filterIsInstance<IrClass>().first()
+        parent = parentClass
+        addDispatchReceiver {
+            startOffset = irField.startOffset
+            endOffset = irField.endOffset
+            type = parentClass.defaultType
+        }
+        builder()
+        annotations += buildSimpleAnnotation(context.irBuiltIns,
+                SYNTHETIC_OFFSET, SYNTHETIC_OFFSET,
+                context.ir.symbols.typedIntrinsic.owner, intrinsicType.name)
+    }
+
+    private fun buildCasFunction(irField: IrField, intrinsicType: IntrinsicType, functionReturnType: IrType) =
+            buildIntrinsicFunction(irField, intrinsicType ) {
+                returnType = functionReturnType
+                addValueParameter {
+                    startOffset = irField.startOffset
+                    endOffset = irField.endOffset
+                    name = Name.identifier("expectedValue")
+                    type = irField.type
+                }
+                addValueParameter {
+                    startOffset = irField.startOffset
+                    endOffset = irField.endOffset
+                    name = Name.identifier("newValue")
+                    type = irField.type
+                }
+            }
+
+    private fun buildAtomicRWMFunction(irField: IrField, intrinsicType: IntrinsicType) =
+            buildIntrinsicFunction(irField, intrinsicType) {
+                returnType = irField.type
+                addValueParameter {
+                    startOffset = irField.startOffset
+                    endOffset = irField.endOffset
+                    name = Name.identifier("value")
+                    type = irField.type
+                }
+            }
+
+
+    private inline fun atomicFunction(irField: IrField, type: NativeMapping.AtomicFunctionType, builder: () -> IrSimpleFunction): IrSimpleFunction {
+        val key = NativeMapping.AtomicFunctionKey(irField, type)
+        return context.mapping.volatileFieldToAtomicFunction.getOrPut(key) {
+            builder().also {
+                context.mapping.functionToVolatileField[it] = irField
+            }
+        }
+    }
+
+    private fun compareAndSetFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.COMPARE_AND_SET) {
+        this.buildCasFunction(irField, IntrinsicType.COMPARE_AND_SET, this.context.irBuiltIns.booleanType)
+    }
+    private fun compareAndSwapFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.COMPARE_AND_SWAP) {
+        this.buildCasFunction(irField, IntrinsicType.COMPARE_AND_SWAP, irField.type)
+    }
+    private fun getAndSetFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.GET_AND_SET) {
+        this.buildAtomicRWMFunction(irField, IntrinsicType.GET_AND_SET)
+    }
+    private fun getAndAddFunction(irField: IrField) = atomicFunction(irField, NativeMapping.AtomicFunctionType.GET_AND_ADD) {
+        this.buildAtomicRWMFunction(irField, IntrinsicType.GET_AND_ADD)
+    }
+
+    private fun IrField.isInteger() = type == context.irBuiltIns.intType ||
+            type == context.irBuiltIns.longType ||
+            type == context.irBuiltIns.shortType ||
+            type == context.irBuiltIns.byteType
+
+    override fun lower(irFile: IrFile) {
+        irFile.transformChildrenVoid(object : IrBuildingTransformer(context) {
+            override fun visitClass(declaration: IrClass): IrStatement {
+                declaration.declarations.transformFlat {
+                    when {
+                        it !is IrProperty -> null
+                        it.backingField?.hasAnnotation(KonanFqNames.volatile) != true -> null
+                        else -> {
+                            val field = it.backingField!!
+                            if (field.type.binaryTypeIsReference() && context.memoryModel != MemoryModel.EXPERIMENTAL) {
+                                it.annotations = it.annotations.filterNot { it.symbol.owner.parentAsClass.hasEqualFqName(KonanFqNames.volatile) }
+                                null
+                            } else {
+                                listOfNotNull(it,
+                                        compareAndSetFunction(field),
+                                        compareAndSwapFunction(field),
+                                        getAndSetFunction(field),
+                                        if (field.isInteger()) getAndAddFunction(field) else null
+                                )
+                            }
+                        }
+                    }
+                }
+                declaration.transformChildrenVoid()
+                return declaration
+            }
+
+            private fun unsupported(message: String) = builder.irCall(context.ir.symbols.throwIllegalArgumentExceptionWithMessage).apply {
+                putValueArgument(0, builder.irString(message))
+            }
+
+            private val intrinsicMap = mapOf(
+                    IntrinsicType.COMPARE_AND_SET_FIELD to ::compareAndSetFunction,
+                    IntrinsicType.COMPARE_AND_SWAP_FIELD to ::compareAndSwapFunction,
+                    IntrinsicType.GET_AND_SET_FIELD to ::getAndSetFunction,
+                    IntrinsicType.GET_AND_ADD_FIELD to ::getAndAddFunction,
+            )
+
+            override fun visitCall(expression: IrCall): IrExpression {
+                expression.transformChildrenVoid(this)
+                val intrinsicType = tryGetIntrinsicType(expression).takeIf { it in intrinsicMap } ?: return expression
+                builder.at(expression)
+                val reference = expression.getValueArgument(0) as? IrPropertyReference
+                        ?: return unsupported("Only compile-time known IrProperties supported for $intrinsicType")
+                val property = reference.symbol.owner
+                val backingField = property.backingField
+                if (backingField?.type?.binaryTypeIsReference() == true && context.memoryModel != MemoryModel.EXPERIMENTAL) {
+                    return unsupported("Only primitives are supported for $intrinsicType with legacy memory model")
+                }
+                if (backingField?.hasAnnotation(KonanFqNames.volatile) != true) {
+                    return unsupported("Only volatile properties are supported for $intrinsicType")
+                }
+                val function = intrinsicMap[intrinsicType]!!(backingField)
+                return builder.irCall(function).apply {
+                    dispatchReceiver = expression.extensionReceiver
+                    putValueArgument(0, expression.getValueArgument(1))
+                    if (intrinsicType == IntrinsicType.COMPARE_AND_SET_FIELD || intrinsicType == IntrinsicType.COMPARE_AND_SWAP_FIELD) {
+                        putValueArgument(1, expression.getValueArgument(2))
+                    }
+                }
+            }
+        })
+    }
+}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.util.OperatorNameConventions
+import org.jetbrains.kotlin.backend.konan.llvm.*
 
 internal class ExternalModulesDFG(val allTypes: List<DataFlowIR.Type.Declared>,
                                   val publicTypes: Map<Long, DataFlowIR.Type.Public>,
@@ -258,7 +259,7 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
 
         context.logMultiple {
             +"SYMBOL TABLE:"
-            symbolTable.classMap.forEach { irClass, type ->
+            symbolTable.classMap.forEach { (irClass, type) ->
                 +"    DESCRIPTOR: ${irClass.descriptor}"
                 +"    TYPE: $type"
                 if (type !is DataFlowIR.Type.Declared)
@@ -309,39 +310,61 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
                     expressions += expression to currentLoop
             }
 
-            if (expression is IrCall && expression.symbol == initInstanceSymbol) {
-                // Skip the constructor call as initInstance is handled specially later.
-                val thiz = expression.getValueArgument(0)!!
-                val constructorCall = expression.getValueArgument(1)!!
-                thiz.acceptVoid(this)
-                constructorCall.acceptChildrenVoid(this)
-                return
+            if (expression is IrCall) {
+                if (expression.symbol == initInstanceSymbol) {
+                    // Skip the constructor call as initInstance is handled specially later.
+                    val thiz = expression.getValueArgument(0)!!
+                    val constructorCall = expression.getValueArgument(1)!!
+                    thiz.acceptVoid(this)
+                    constructorCall.acceptChildrenVoid(this)
+                    return
+                }
+                if (expression.symbol == executeImplSymbol) {
+                    // Producer and job of executeImpl are called externally, we need to reflect this somehow.
+                    val producerInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset,
+                            executeImplProducerInvoke.returnType,
+                            executeImplProducerInvoke.symbol,
+                            executeImplProducerInvoke.symbol.owner.typeParameters.size,
+                            executeImplProducerInvoke.symbol.owner.valueParameters.size,
+                            STATEMENT_ORIGIN_PRODUCER_INVOCATION)
+                    producerInvocation.dispatchReceiver = expression.getValueArgument(2)
+
+                    expressions += producerInvocation to currentLoop
+
+                    val jobFunctionReference = expression.getValueArgument(3) as? IrFunctionReference
+                            ?: error("A function reference expected")
+                    val jobInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset,
+                            jobFunctionReference.symbol.owner.returnType,
+                            jobFunctionReference.symbol as IrSimpleFunctionSymbol,
+                            jobFunctionReference.symbol.owner.typeParameters.size,
+                            jobFunctionReference.symbol.owner.valueParameters.size,
+                            STATEMENT_ORIGIN_JOB_INVOCATION)
+                    jobInvocation.putValueArgument(0, producerInvocation)
+
+                    expressions += jobInvocation to currentLoop
+                }
+                val intrinsicType = tryGetIntrinsicType(expression)
+                if (intrinsicType == IntrinsicType.COMPARE_AND_SET || intrinsicType == IntrinsicType.COMPARE_AND_SWAP) {
+                    expressions += IrSetFieldImpl(
+                            expression.startOffset, expression.endOffset,
+                            context.mapping.functionToVolatileField[expression.symbol.owner]!!.symbol,
+                            expression.dispatchReceiver,
+                            expression.getValueArgument(1)!!,
+                            context.irBuiltIns.unitType
+                    ) to currentLoop
+                }
+                if (intrinsicType == IntrinsicType.GET_AND_SET) {
+                    expressions += IrSetFieldImpl(
+                            expression.startOffset, expression.endOffset,
+                            context.mapping.functionToVolatileField[expression.symbol.owner]!!.symbol,
+                            expression.dispatchReceiver,
+                            expression.getValueArgument(0)!!,
+                            context.irBuiltIns.unitType
+                    ) to currentLoop
+                }
             }
 
-            if (expression is IrCall && expression.symbol == executeImplSymbol) {
-                // Producer and job of executeImpl are called externally, we need to reflect this somehow.
-                val producerInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset,
-                        executeImplProducerInvoke.returnType,
-                        executeImplProducerInvoke.symbol,
-                        executeImplProducerInvoke.symbol.owner.typeParameters.size,
-                        executeImplProducerInvoke.symbol.owner.valueParameters.size,
-                        STATEMENT_ORIGIN_PRODUCER_INVOCATION)
-                producerInvocation.dispatchReceiver = expression.getValueArgument(2)
 
-                expressions += producerInvocation to currentLoop
-
-                val jobFunctionReference = expression.getValueArgument(3) as? IrFunctionReference
-                        ?: error("A function reference expected")
-                val jobInvocation = IrCallImpl.fromSymbolDescriptor(expression.startOffset, expression.endOffset,
-                        jobFunctionReference.symbol.owner.returnType,
-                        jobFunctionReference.symbol as IrSimpleFunctionSymbol,
-                        jobFunctionReference.symbol.owner.typeParameters.size,
-                        jobFunctionReference.symbol.owner.valueParameters.size,
-                        STATEMENT_ORIGIN_JOB_INVOCATION)
-                jobInvocation.putValueArgument(0, producerInvocation)
-
-                expressions += jobInvocation to currentLoop
-            }
 
             // TODO: A little bit hacky but it is the simplest solution.
             // See ObjC instanceOf code generation for details.

kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp

@@ -34,6 +34,10 @@ touchFunction(AllocArrayInstance)
 touchFunction(InitAndRegisterGlobal)
 touchFunction(UpdateHeapRef)
 touchFunction(UpdateStackRef)
+touchFunction(UpdateVolatileHeapRef)
+touchFunction(CompareAndSwapVolatileHeapRef)
+touchFunction(CompareAndSetVolatileHeapRef)
+touchFunction(GetAndSetVolatileHeapRef)
 touchFunction(UpdateReturnRef)
 touchFunction(ZeroHeapRef)
 touchFunction(ZeroArrayRefs)

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -3343,6 +3343,20 @@ RUNTIME_NOTHROW void UpdateHeapRefRelaxed(ObjHeader** location, const ObjHeader*
   updateHeapRef<false>(location, object);
 }
 
+ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) {
+    RuntimeFail("Shouldn't be used");
+}
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
+    RuntimeFail("Shouldn't be used");
+}
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
+    RuntimeFail("Shouldn't be used");
+}
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) {
+    RuntimeFail("Shouldn't be used");
+}
+
+
 RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArrayStrict(const ArrayHeader* array, int fromIndex, int toIndex,
                                                         int count) {
   updateHeapRefsInsideOneArray<true>(array, fromIndex, toIndex, count);

kotlin-native/runtime/src/main/cpp/Atomic.cpp

@@ -29,41 +29,6 @@ struct AtomicReferenceLayout {
   KInt cookie_;
 };
 
-template<typename T> struct AtomicPrimitive {
-  ObjHeader header;
-  volatile T value_;
-};
-
-template <typename T> inline volatile T* getValueLocation(KRef thiz) {
-  AtomicPrimitive<T>* atomic = reinterpret_cast<AtomicPrimitive<T>*>(thiz);
-  return &atomic->value_;
-}
-
-template <typename T> void setImpl(KRef thiz, T value) {
-  volatile T* location = getValueLocation<T>(thiz);
-  atomicSet(location, value);
-}
-
-template <typename T> T getImpl(KRef thiz) {
-  volatile T* location = getValueLocation<T>(thiz);
-  return atomicGet(location);
-}
-
-template <typename T> T addAndGetImpl(KRef thiz, T delta) {
-  volatile T* location = getValueLocation<T>(thiz);
-  return atomicAdd(location, delta);
-}
-
-template <typename T> T compareAndSwapImpl(KRef thiz, T expectedValue, T newValue) {
-  volatile T* location = getValueLocation<T>(thiz);
-  return compareAndSwap(location, expectedValue, newValue);
-}
-
-template <typename T> KBoolean compareAndSetImpl(KRef thiz, T expectedValue, T newValue) {
-    volatile T* location = getValueLocation<T>(thiz);
-    return compareAndSet(location, expectedValue, newValue);
-}
-
 inline AtomicReferenceLayout* asAtomicReference(KRef thiz) {
     return reinterpret_cast<AtomicReferenceLayout*>(thiz);
 }
@@ -72,119 +37,7 @@ inline AtomicReferenceLayout* asAtomicReference(KRef thiz) {
 
 extern "C" {
 
-KInt Kotlin_AtomicInt_addAndGet(KRef thiz, KInt delta) {
-    return addAndGetImpl(thiz, delta);
-}
 
-KInt Kotlin_AtomicInt_compareAndSwap(KRef thiz, KInt expectedValue, KInt newValue) {
-    return compareAndSwapImpl(thiz, expectedValue, newValue);
-}
-
-KBoolean Kotlin_AtomicInt_compareAndSet(KRef thiz, KInt expectedValue, KInt newValue) {
-    return compareAndSetImpl(thiz, expectedValue, newValue);
-}
-
-void Kotlin_AtomicInt_set(KRef thiz, KInt newValue) {
-    setImpl(thiz, newValue);
-}
-
-KInt Kotlin_AtomicInt_get(KRef thiz) {
-    return getImpl<KInt>(thiz);
-}
-
-#if KONAN_NO_64BIT_ATOMIC
-static int lock64 = 0;
-#endif
-
-KLong Kotlin_AtomicLong_addAndGet(KRef thiz, KLong delta) {
-#if KONAN_NO_64BIT_ATOMIC
-    // Potentially huge performance penalty, but correct.
-    while (compareAndSwap(&lock64, 0, 1) != 0);
-    volatile KLong* address = getValueLocation<KLong>(thiz);
-    KLong old = *address;
-    KLong newValue = old + delta;
-    *address = newValue;
-    compareAndSwap(&lock64, 1, 0);
-    return newValue;
-#else
-    return addAndGetImpl(thiz, delta);
-#endif
-}
-
-KLong Kotlin_AtomicLong_compareAndSwap(KRef thiz, KLong expectedValue, KLong newValue) {
-#if KONAN_NO_64BIT_ATOMIC
-    // Potentially huge performance penalty, but correct.
-    while (compareAndSwap(&lock64, 0, 1) != 0);
-    volatile KLong* address = getValueLocation<KLong>(thiz);
-    KLong old = *address;
-    if (old == expectedValue) {
-      *address = newValue;
-    }
-    compareAndSwap(&lock64, 1, 0);
-    return old;
-#else
-    return compareAndSwapImpl(thiz, expectedValue, newValue);
-#endif
-}
-
-KBoolean Kotlin_AtomicLong_compareAndSet(KRef thiz, KLong expectedValue, KLong newValue) {
-#if KONAN_NO_64BIT_ATOMIC
-    // Potentially huge performance penalty, but correct.
-    KBoolean result = false;
-    while (compareAndSwap(&lock64, 0, 1) != 0);
-    volatile KLong* address = getValueLocation<KLong>(thiz);
-    KLong old = *address;
-    if (old == expectedValue) {
-      result = true;
-      *address = newValue;
-    }
-    compareAndSwap(&lock64, 1, 0);
-    return result;
-#else
-    return compareAndSetImpl(thiz, expectedValue, newValue);
-#endif
-}
-
-void Kotlin_AtomicLong_set(KRef thiz, KLong newValue) {
-#if KONAN_NO_64BIT_ATOMIC
-    // Potentially huge performance penalty, but correct.
-    while (compareAndSwap(&lock64, 0, 1) != 0);
-    volatile KLong* address = getValueLocation<KLong>(thiz);
-    *address = newValue;
-    compareAndSwap(&lock64, 1, 0);
-#else
-    setImpl(thiz, newValue);
-#endif
-}
-
-KLong Kotlin_AtomicLong_get(KRef thiz) {
-#if KONAN_NO_64BIT_ATOMIC
-    // Potentially huge performance penalty, but correct.
-    while (compareAndSwap(&lock64, 0, 1) != 0);
-    volatile KLong* address = getValueLocation<KLong>(thiz);
-    KLong value = *address;
-    compareAndSwap(&lock64, 1, 0);
-    return value;
-#else
-    return getImpl<KLong>(thiz);
-#endif
-}
-
-KNativePtr Kotlin_AtomicNativePtr_compareAndSwap(KRef thiz, KNativePtr expectedValue, KNativePtr newValue) {
-    return compareAndSwapImpl(thiz, expectedValue, newValue);
-}
-
-KBoolean Kotlin_AtomicNativePtr_compareAndSet(KRef thiz, KNativePtr expectedValue, KNativePtr newValue) {
-    return compareAndSetImpl(thiz, expectedValue, newValue);
-}
-
-void Kotlin_AtomicNativePtr_set(KRef thiz, KNativePtr newValue) {
-    setImpl(thiz, newValue);
-}
-
-KNativePtr Kotlin_AtomicNativePtr_get(KRef thiz) {
-    return getImpl<KNativePtr>(thiz);
-}
 
 void Kotlin_AtomicReference_checkIfFrozen(KRef value) {
     if (!kotlin::compiler::freezingEnabled()) {

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -247,6 +247,12 @@ void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW;
 void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates heap/static data location.
 void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
+// Updates volatile heap/static data location.
+void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
+OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) RUNTIME_NOTHROW;
+bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) RUNTIME_NOTHROW;
+OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) RUNTIME_NOTHROW;
+
 // Updates heap/static data in one array.
 void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW;
 // Updates location if it is null, atomically.

kotlin-native/runtime/src/main/kotlin/kotlin/native/concurrent/Atomics.kt

@@ -7,6 +7,8 @@ package kotlin.native.concurrent
 
 import kotlinx.cinterop.NativePtr
 import kotlin.native.internal.*
+import kotlin.reflect.*
+import kotlin.concurrent.*
 
 /**
  * Wrapper around [Int] with atomic synchronized operations.
@@ -16,23 +18,15 @@ import kotlin.native.internal.*
  * So shared frozen objects can have mutable fields of [AtomicInt] type.
  */
 @Frozen
-@OptIn(FreezingIsDeprecated::class)
-public class AtomicInt(private var value_: Int) {
-    /**
-     * The value being held by this class.
-     */
-    public var value: Int
-            get() = getImpl()
-            set(new) = setImpl(new)
-
+@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class)
+public class AtomicInt(public @Volatile var value: Int) {
     /**
      * Increments the value by [delta] and returns the new value.
      *
      * @param delta the value to add
      * @return the new value
      */
-    @GCUnsafeCall("Kotlin_AtomicInt_addAndGet")
-    external public fun addAndGet(delta: Int): Int
+    public fun addAndGet(delta: Int): Int = getAndAddField(AtomicInt::value, delta) + delta
 
     /**
      * Compares value with [expected] and replaces it with [new] value if values matches.
@@ -41,8 +35,7 @@ public class AtomicInt(private var value_: Int) {
      * @param new the new value
      * @return the old value
      */
-    @GCUnsafeCall("Kotlin_AtomicInt_compareAndSwap")
-    external public fun compareAndSwap(expected: Int, new: Int): Int
+    public fun compareAndSwap(expected: Int, new: Int): Int = compareAndSwapField(AtomicInt::value, expected, new)
 
     /**
      * Compares value with [expected] and replaces it with [new] value if values matches.
@@ -51,8 +44,7 @@ public class AtomicInt(private var value_: Int) {
      * @param new the new value
      * @return true if successful
      */
-    @GCUnsafeCall("Kotlin_AtomicInt_compareAndSet")
-    external public fun compareAndSet(expected: Int, new: Int): Boolean
+    public fun compareAndSet(expected: Int, new: Int): Boolean = compareAndSetField(AtomicInt::value, expected, new)
 
     /**
      * Increments value by one.
@@ -74,13 +66,6 @@ public class AtomicInt(private var value_: Int) {
      * @return the string representation
      */
     public override fun toString(): String = value.toString()
-
-    // Implementation details.
-    @GCUnsafeCall("Kotlin_AtomicInt_set")
-    private external fun setImpl(new: Int): Unit
-
-    @GCUnsafeCall("Kotlin_AtomicInt_get")
-    private external fun getImpl(): Int
 }
 
 /**
@@ -91,14 +76,8 @@ public class AtomicInt(private var value_: Int) {
  * So shared frozen objects can have mutable fields of [AtomicLong] type.
  */
 @Frozen
-@OptIn(FreezingIsDeprecated::class)
-public class AtomicLong(private var value_: Long = 0)  {
-    /**
-     * The value being held by this class.
-     */
-    public var value: Long
-        get() = getImpl()
-        set(new) = setImpl(new)
+@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class)
+public class AtomicLong(public @Volatile var value: Long = 0)  {
 
     /**
      * Increments the value by [delta] and returns the new value.
@@ -106,8 +85,7 @@ public class AtomicLong(private var value_: Long = 0)  {
      * @param delta the value to add
      * @return the new value
      */
-    @GCUnsafeCall("Kotlin_AtomicLong_addAndGet")
-    external public fun addAndGet(delta: Long): Long
+    public fun addAndGet(delta: Long): Long = getAndAddField(AtomicLong::value, delta) + delta
 
     /**
      * Increments the value by [delta] and returns the new value.
@@ -124,8 +102,7 @@ public class AtomicLong(private var value_: Long = 0)  {
      * @param new the new value
      * @return the old value
      */
-    @GCUnsafeCall("Kotlin_AtomicLong_compareAndSwap")
-    external public fun compareAndSwap(expected: Long, new: Long): Long
+    public fun compareAndSwap(expected: Long, new: Long): Long = compareAndSwapField(AtomicLong::value, expected, new)
 
     /**
      * Compares value with [expected] and replaces it with [new] value if values matches.
@@ -134,8 +111,7 @@ public class AtomicLong(private var value_: Long = 0)  {
      * @param new the new value
      * @return true if successful, false if state is unchanged
      */
-    @GCUnsafeCall("Kotlin_AtomicLong_compareAndSet")
-    external public fun compareAndSet(expected: Long, new: Long): Boolean
+    public fun compareAndSet(expected: Long, new: Long): Boolean = compareAndSetField(AtomicLong::value, expected, new)
 
     /**
      * Increments value by one.
@@ -157,13 +133,6 @@ public class AtomicLong(private var value_: Long = 0)  {
      * @return the string representation of this object
      */
     public override fun toString(): String = value.toString()
-
-    // Implementation details.
-    @GCUnsafeCall("Kotlin_AtomicLong_set")
-    private external fun setImpl(new: Long): Unit
-
-    @GCUnsafeCall("Kotlin_AtomicLong_get")
-    private external fun getImpl(): Long
 }
 
 /**
@@ -174,14 +143,8 @@ public class AtomicLong(private var value_: Long = 0)  {
  * So shared frozen objects can have mutable fields of [AtomicNativePtr] type.
  */
 @Frozen
-@OptIn(FreezingIsDeprecated::class)
-public class AtomicNativePtr(private var value_: NativePtr) {
-    /**
-     * The value being held by this class.
-     */
-    public var value: NativePtr
-        get() = getImpl()
-        set(new) = setImpl(new)
+@OptIn(FreezingIsDeprecated::class, ExperimentalStdlibApi::class)
+public class AtomicNativePtr(public @Volatile var value: NativePtr) {
 
     /**
      * Compares value with [expected] and replaces it with [new] value if values matches.
@@ -190,8 +153,8 @@ public class AtomicNativePtr(private var value_: NativePtr) {
      * @param new the new value
      * @return the old value
      */
-    @GCUnsafeCall("Kotlin_AtomicNativePtr_compareAndSwap")
-    external public fun compareAndSwap(expected: NativePtr, new: NativePtr): NativePtr
+    public fun compareAndSwap(expected: NativePtr, new: NativePtr): NativePtr =
+            compareAndSwapField(AtomicNativePtr::value, expected, new)
 
     /**
      * Compares value with [expected] and replaces it with [new] value if values matches.
@@ -200,8 +163,8 @@ public class AtomicNativePtr(private var value_: NativePtr) {
      * @param new the new value
      * @return true if successful
      */
-    @GCUnsafeCall("Kotlin_AtomicNativePtr_compareAndSet")
-    external public fun compareAndSet(expected: NativePtr, new: NativePtr): Boolean
+    public fun compareAndSet(expected: NativePtr, new: NativePtr): Boolean =
+            compareAndSetField(AtomicNativePtr::value, expected, new)
 
     /**
      * Returns the string representation of this object.
@@ -209,13 +172,6 @@ public class AtomicNativePtr(private var value_: NativePtr) {
      * @return string representation of this object
      */
     public override fun toString(): String = value.toString()
-
-    // Implementation details.
-    @GCUnsafeCall("Kotlin_AtomicNativePtr_set")
-    private external fun setImpl(new: NativePtr): Unit
-
-    @GCUnsafeCall("Kotlin_AtomicNativePtr_get")
-    private external fun getImpl(): NativePtr
 }
 
 
@@ -431,3 +387,134 @@ public class FreezableAtomicReference<T>(private var value_: T) {
     @GCUnsafeCall("Kotlin_AtomicReference_compareAndSet")
     private external fun compareAndSetImpl(expected: Any?, new: Any?): Boolean
 }
+
+
+/**
+ * Compares the value of the field referenced by [fieldRef] from [this] object to [expectedValue], and if they are equal,
+ * atomically replaces it with [newValue].
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * Comparison is done by reference or value depending on field representation.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Returns true if the actual field value matched [expectedValue]
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.COMPARE_AND_SET_FIELD)
+internal external fun <T, S> T.compareAndSetField(filedRef: KMutableProperty1<T, S>, expectedValue: S, newValue: S): Boolean
+
+/**
+ * Compares the value of the field referenced by [fieldRef] from [this] object to [expectedValue], and if they are equal,
+ * atomically replaces it with [newValue].
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * Comparison is done by reference or value depending on field representation.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Returns true if the actual field value before operation.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.COMPARE_AND_SWAP_FIELD)
+internal external fun <T, S> T.compareAndSwapField(filedRef: KMutableProperty1<T, S>, expectedValue: S, newValue: S): S
+
+/**
+ * Atomically sets value of the field referenced by [fieldRef] from [this] object to [newValue] and returns old field value.
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.GET_AND_SET_FIELD)
+internal external fun <T, S> T.getAndSetField(filedRef: KMutableProperty1<T, S>, newValue: S): S
+
+
+/**
+ * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value.
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD)
+internal external fun <T> T.getAndAddField(filedRef: KMutableProperty1<T, Short>, delta: Short): Short
+
+/**
+ * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value.
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD)
+internal external fun <T> T.getAndAddField(filedRef: KMutableProperty1<T, Int>, newValue: Int): Int
+
+/**
+ * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value.
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD)
+internal external fun <T> T.getAndAddField(filedRef: KMutableProperty1<T, Long>, newValue: Long): Long
+
+/**
+ * Atomically increments value of the field referenced by [fieldRef] from [this] object by [delta] and returns old field value.
+ *
+ * For now, it can be used only within the same file, where class [T] is defined.
+ * Check https://youtrack.jetbrains.com/issue/KT-55426 for details.
+ *
+ * If [fieldRef] is not a compile-time known reference to the property with [Volatile] annotation [IllegalArgumentException]
+ * would be thrown.
+ *
+ * If property referenced by [fieldRef] has nontrivial setter it will not be called.
+ *
+ * Legacy MM: if [fieldRef] is a reference for a non-value represented field, [IllegalArgumentException] would be thrown.
+ */
+@PublishedApi
+@TypedIntrinsic(IntrinsicType.GET_AND_ADD_FIELD)
+internal external fun <T> T.getAndAddField(filedRef: KMutableProperty1<T, Byte>, newValue: Byte): Byte

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt

@@ -80,5 +80,15 @@ class IntrinsicType {
 
         // Worker
         const val WORKER_EXECUTE                = "WORKER_EXECUTE"
+
+        // Atomic
+        const val COMPARE_AND_SET_FIELD         = "COMPARE_AND_SET_FIELD"
+        const val COMPARE_AND_SWAP_FIELD        = "COMPARE_AND_SWAP_FIELD"
+        const val GET_AND_SET_FIELD             = "GET_AND_SET_FIELD"
+        const val GET_AND_ADD_FIELD             = "GET_AND_ADD_FIELD"
+        const val COMPARE_AND_SET               = "COMPARE_AND_SET"
+        const val COMPARE_AND_SWAP              = "COMPARE_AND_SWAP"
+        const val GET_AND_SET                   = "GET_AND_SET"
+        const val GET_AND_ADD                   = "GET_AND_ADD"
     }
 }

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -188,6 +188,22 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location
     mm::SetHeapRef(location, const_cast<ObjHeader*>(object));
 }
 
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) {
+    mm::SetHeapRefAtomicSeqCst(location, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
+    RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
+    return mm::CompareAndSetHeapRef(location, expectedValue, newValue);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) {
+    RETURN_RESULT_OF(mm::GetAndSetHeapRef, location, newValue);
+}
+
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
     if (object == nullptr) return;
     ObjHeader* result = nullptr; // No need to store this value in a rootset.

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -34,6 +34,12 @@ ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value)
     __atomic_store_n(location, value, __ATOMIC_RELEASE);
 }
 
+ALWAYS_INLINE void mm::SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept {
+    AssertThreadState(ThreadState::kRunnable);
+    __atomic_store_n(location, value, __ATOMIC_SEQ_CST);
+}
+
+
 ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept {
     AssertThreadState(ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
@@ -45,14 +51,24 @@ ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHea
     AssertThreadState(ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
     ObjHeader* actual = expected;
-    // TODO: Do we need this strong memory model? Do we need to use strong CAS?
-    // This intrinsic modifies `actual` non-atomically.
     __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
-    // On success, we already have old value (== `expected`) in `actual`.
-    // On failure, we have the old value written into `actual`.
     RETURN_OBJ(actual);
 }
 
+ALWAYS_INLINE bool mm::CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept {
+    AssertThreadState(ThreadState::kRunnable);
+    // TODO: Make this work with GCs that can stop thread at any point.
+    ObjHeader* actual = expected;
+    return __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept {
+    AssertThreadState(ThreadState::kRunnable);;
+    auto *actual = __atomic_exchange_n(location, value,  __ATOMIC_SEQ_CST);
+    RETURN_OBJ(actual);
+}
+
+
 #pragma clang diagnostic pop
 
 OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept {

kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp

@@ -24,8 +24,11 @@ class ThreadData;
 void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept;
 void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept;
 void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept;
+void SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept;
 OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept;
 OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept;
+bool CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept;
+OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept;
 OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept;
 OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;