azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Introduce intrinsics that atomically update array elements

Browse Source 
Supported atomic update of elements for IntArray, LongArray and Array<T>
See KT-58360

Merge-request: KT-MR-11020
Merged-by: Maria Sokolova <maria.sokolova@jetbrains.com>
This commit is contained in:
mvicsokolova
2023-07-12 14:32:36 +00:00
committed by Space Team
parent 20c53fc15d
commit d9fa9c1b3b
13 changed files with 598 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/testData/codegen/box/volatile/atomicArrayIntrinsics.kt
Vendored
+235
View File
@@ -0,0 +1,235 @@
// TARGET_BACKEND: NATIVE
@file:Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
@file:OptIn(kotlin.ExperimentalStdlibApi::class)
import kotlin.native.concurrent.*
import kotlin.concurrent.*
import kotlin.native.internal.*
import kotlin.test.*
private val intArrStatic = IntArray(10) { i: Int -> i * 10 }
private val longArrStatic = LongArray(10) { i: Int -> i * 10L }
private val refArrStatic = arrayOfNulls<String?>(10)
private class ArrayIntrinsicsSmokeTest {
val intArr = IntArray(10) { i: Int -> i * 10 }
val longArr = LongArray(10) { i: Int -> i * 10L }
val refArr = arrayOfNulls<String?>(10)
fun testIntArrayIntrinsics() {
// atomicGet
val res = intArr.atomicGet(2)
assertEquals(20, res, "IntArray: FAIL 1")
// compareAndExchange
val res1 = intArr.compareAndExchange(2, 20, 222) // success
assertTrue(res1 == 20 && intArr[2] == 222, "IntArray: FAIL 2")
val res2 = intArr.compareAndExchange(2, 222, 2222) // success
assertTrue(res2 == 222 && intArr[2] == 2222, "IntArray: FAIL 3")
val res3 = intArr.compareAndExchange(2, 223, 22222) // should fail
assertTrue(res3 == 2222 && intArr[2] == 2222, "IntArray: FAIL 4")
val res4 = intArr.compareAndExchange(9, 10, 999) // should fail
assertTrue(res4 == 90 && intArr[9] == 90, "IntArray: FAIL 5")
// getAndSet
assertEquals(2222, intArr.getAndSet(2, 20), "IntArray: FAIL 6")
assertEquals(20, intArr.getAndSet(2, 200), "IntArray: FAIL 7")
assertEquals(200, intArr.atomicGet(2), "IntArray: FAIL 8")
// atomicSet
intArr.atomicSet(1, 111)
assertEquals(111, intArr.atomicGet(1), "IntArray: FAIL 9")
// getAndAdd
assertEquals(111, intArr.getAndAdd(1, 100), "IntArray: FAIL 10")
assertEquals(211, intArr.atomicGet(1), "IntArray: FAIL 11")
assertEquals(211, intArr.getAndAdd(1, -100), "IntArray: FAIL 12")
assertEquals(111, intArr.atomicGet(1), "IntArray: FAIL 13")
// compareAndSet
val aaa = 1111
intArr.atomicSet(1, aaa)
assertTrue(intArr.compareAndSet(1, 1111, 2222), "IntArray: FAIL 14")
assertFalse(intArr.compareAndSet(1, 1111, 3333), "IntArray: FAIL 15")
assertTrue(intArr.atomicGet(1) == 2222, "IntArray: FAIL 16")
}
fun testLongArrayIntrinsics() {
// atomicGet
val res = longArr.atomicGet(2)
assertEquals(20L, res, "IntArray: FAIL 1")
// compareAndExchange
val res1 = longArr.compareAndExchange(2, 20L, 222L) // success
assertTrue(res1 == 20L && longArr[2] == 222L, "LongArray: FAIL 2")
val res2 = longArr.compareAndExchange(2, 222L, 2222L) // success
assertTrue(res2 == 222L && longArr[2] == 2222L, "LongArray: FAIL 3")
val res3 = longArr.compareAndExchange(2, 223L, 22222L) // should fail
assertTrue(res3 == 2222L && longArr[2] == 2222L, "LongArray: FAIL 4")
val res4 = longArr.compareAndExchange(9, 10L, 999L) // should fail
assertTrue(res4 == 90L && longArr[9] == 90L, "LongArray: FAIL 5")
// getAndSet
assertEquals(2222L, longArr.getAndSet(2, 20L), "LongArray: FAIL 6")
assertEquals(20L, longArr.getAndSet(2, 200L), "LongArray: FAIL 7")
assertEquals(200L, longArr.atomicGet(2), "LongArray: FAIL 8")
// atomicSet
longArr.atomicSet(1, 111L)
assertEquals(111L, longArr.atomicGet(1), "LongArray: FAIL 9")
// getAndAdd
assertEquals(111L, longArr.getAndAdd(1, 100L), "LongArray: FAIL 10")
assertEquals(211L, longArr.atomicGet(1), "LongArray: FAIL 11")
assertEquals(211L, longArr.getAndAdd(1, -100L), "LongArray: FAIL 12")
assertEquals(111L, longArr.atomicGet(1), "LongArray: FAIL 13")
// compareAndSet
val aaa = 1111L
longArr.atomicSet(1, aaa)
assertTrue(longArr.compareAndSet(1, 1111L, 2222L), "LongArray: FAIL 14")
assertFalse(longArr.compareAndSet(1, 1111L, 3333L), "LongArray: FAIL 15")
assertTrue(longArr.atomicGet(1) == 2222L, "LongArray: FAIL 16")
}
fun testReferenceArrayIntrinsics() {
// atomicGet
assertTrue(refArr.atomicGet(0) == refArr[0], "Array: FAIL 1")
// compareAndExchange
val newValue = "aaa"
val res1 = refArr.compareAndExchange(3, null, newValue)
assertTrue(res1 == null && refArr[3] == newValue, "Array: FAIL 2")
val res2 = refArr.compareAndExchange(3, newValue, "bbb")
assertTrue(res2 == newValue && refArr[3] == "bbb", "Array: FAIL 3")
val res3 = refArr.compareAndExchange(3, newValue, "ccc")
assertTrue(res3 == "bbb" && refArr[3] == "bbb", "Array: FAIL 4")
// getAndSet
val res4 = refArr.getAndSet(4, "aaa")
assertEquals(null, res4, "Array: FAIL 5")
val res5 = refArr.getAndSet(4, "bbb")
assertEquals("aaa", res5, "Array: FAIL 6")
//set
refArr.atomicSet(1, "aaa")
assertEquals("aaa", refArr.atomicGet(1), "Array: FAIL 7")
// compareAndSet
val aaa = "aaa"
refArr.atomicSet(1, aaa)
assertTrue(refArr.compareAndSet(1, aaa, "aaa1"), "Array: FAIL 8")
assertFalse(refArr.compareAndSet(1, aaa, "aaa1"), "Array: FAIL 9")
assertEquals("aaa1", refArr.atomicGet(1), "Array: FAIL 10")
}
// Tests that there are no restrictions for the array property as the receiver of the intrinsic call (it can be unknown at compile-time)
fun testComileTimeUnknownArrayReceiver() {
intArr.extensionArrayUpdater()
arrayArgumentUpdater(intArr)
}
private fun IntArray.extensionArrayUpdater() {
atomicSet(1, 45)
assertTrue(compareAndSet(1, 45, 50))
assertEquals(50, atomicGet(1))
}
private fun arrayArgumentUpdater(arr: IntArray) {
arr.atomicSet(1, 45)
assertTrue(arr.compareAndSet(1, 45, 50))
assertEquals(50, arr.atomicGet(1))
}
}
fun testStaticIntArrayIntrinsics() {
// atomicGet
val res = intArrStatic.atomicGet(2)
assertEquals(20, res, "static IntArray: FAIL 1")
// compareAndExchange
val res1 = intArrStatic.compareAndExchange(2, 20, 222) // success
assertTrue(res1 == 20 && intArrStatic[2] == 222, "static IntArray: FAIL 2")
val res2 = intArrStatic.compareAndExchange(2, 222, 2222) // success
assertTrue(res2 == 222 && intArrStatic[2] == 2222, "static IntArray: FAIL 3")
val res3 = intArrStatic.compareAndExchange(2, 223, 22222) // should fail
assertTrue(res3 == 2222 && intArrStatic[2] == 2222, "static IntArray: FAIL 4")
val res4 = intArrStatic.compareAndExchange(9, 10, 999) // should fail
assertTrue(res4 == 90 && intArrStatic[9] == 90, "static IntArray: FAIL 5")
// getAndSet
assertEquals(2222, intArrStatic.getAndSet(2, 20), "static IntArray: FAIL 6")
assertEquals(20, intArrStatic.getAndSet(2, 200), "static IntArray: FAIL 7")
assertEquals(200, intArrStatic.atomicGet(2), "static IntArray: FAIL 8")
// atomicSet
intArrStatic.atomicSet(1, 111)
assertEquals(111, intArrStatic.atomicGet(1), "static IntArray: FAIL 9")
// getAndAdd
assertEquals(111, intArrStatic.getAndAdd(1, 100), "static IntArray: FAIL 10")
assertEquals(211, intArrStatic.atomicGet(1), "static IntArray: FAIL 11")
assertEquals(211, intArrStatic.getAndAdd(1, -100), "static IntArray: FAIL 12")
assertEquals(111, intArrStatic.atomicGet(1), "static IntArray: FAIL 13")
// compareAndSet
val aaa = 1111
intArrStatic.atomicSet(1, aaa)
assertTrue(intArrStatic.compareAndSet(1, 1111, 2222), "static IntArray: FAIL 14")
assertFalse(intArrStatic.compareAndSet(1, 1111, 3333), "static IntArray: FAIL 15")
assertTrue(intArrStatic.atomicGet(1) == 2222, "static IntArray: FAIL 16")
}
fun testStaticLongArrayIntrinsics() {
// atomicGet
val res = longArrStatic.atomicGet(2)
assertEquals(20L, res, "IntArray: FAIL 1")
// compareAndExchange
val res1 = longArrStatic.compareAndExchange(2, 20L, 222L) // success
assertTrue(res1 == 20L && longArrStatic[2] == 222L, "LongArray: FAIL 2")
val res2 = longArrStatic.compareAndExchange(2, 222L, 2222L) // success
assertTrue(res2 == 222L && longArrStatic[2] == 2222L, "LongArray: FAIL 3")
val res3 = longArrStatic.compareAndExchange(2, 223L, 22222L) // should fail
assertTrue(res3 == 2222L && longArrStatic[2] == 2222L, "LongArray: FAIL 4")
val res4 = longArrStatic.compareAndExchange(9, 10L, 999L) // should fail
assertTrue(res4 == 90L && longArrStatic[9] == 90L, "LongArray: FAIL 5")
// getAndSet
assertEquals(2222L, longArrStatic.getAndSet(2, 20L), "LongArray: FAIL 6")
assertEquals(20L, longArrStatic.getAndSet(2, 200L), "LongArray: FAIL 7")
assertEquals(200L, longArrStatic.atomicGet(2), "LongArray: FAIL 8")
// atomicSet
longArrStatic.atomicSet(1, 111L)
assertEquals(111L, longArrStatic.atomicGet(1), "LongArray: FAIL 9")
// getAndAdd
assertEquals(111L, longArrStatic.getAndAdd(1, 100L), "LongArray: FAIL 10")
assertEquals(211L, longArrStatic.atomicGet(1), "LongArray: FAIL 11")
assertEquals(211L, longArrStatic.getAndAdd(1, -100L), "LongArray: FAIL 12")
assertEquals(111L, longArrStatic.atomicGet(1), "LongArray: FAIL 13")
// compareAndSet
val aaa = 1111L
longArrStatic.atomicSet(1, aaa)
assertTrue(longArrStatic.compareAndSet(1, 1111L, 2222L), "LongArray: FAIL 14")
assertFalse(longArrStatic.compareAndSet(1, 1111L, 3333L), "LongArray: FAIL 15")
assertTrue(longArrStatic.atomicGet(1) == 2222L, "LongArray: FAIL 16")
}
fun testStaticReferenceArrayIntrinsics() {
// atomicGet
assertTrue(refArrStatic.atomicGet(0) == refArrStatic[0], "static Array: FAIL 1")
// compareAndExchange
val newValue = "aaa"
val res1 = refArrStatic.compareAndExchange(3, null, newValue)
assertTrue(res1 == null && refArrStatic[3] == newValue, "static Array: FAIL 2")
val res2 = refArrStatic.compareAndExchange(3, newValue, "bbb")
assertTrue(res2 == newValue && refArrStatic[3] == "bbb", "static Array: FAIL 3")
val res3 = refArrStatic.compareAndExchange(3, newValue, "ccc")
assertTrue(res3 == "bbb" && refArrStatic[3] == "bbb", "static Array: FAIL 4")
// getAndSet
val res4 = refArrStatic.getAndSet(4, "aaa")
assertEquals(null, res4, "static Array: FAIL 5")
val res5 = refArrStatic.getAndSet(4, "bbb")
assertEquals("aaa", res5, "static Array: FAIL 6")
// atomicSet
refArrStatic.atomicSet(1, "aaa")
assertEquals("aaa", refArrStatic.atomicGet(1), "static Array: FAIL 7")
// compareAndSet
val aaa = "aaa"
refArrStatic.atomicSet(1, aaa)
assertTrue(refArrStatic.compareAndSet(1, aaa, "aaa1"), "static Array: FAIL 8")
assertFalse(refArrStatic.compareAndSet(1, aaa, "aaa1"), "static Array: FAIL 9")
assertEquals("aaa1", refArrStatic.atomicGet(1), "static Array: FAIL 10")
}
fun box() : String {
val testClass = ArrayIntrinsicsSmokeTest()
testClass.testIntArrayIntrinsics()
testClass.testLongArrayIntrinsics()
testClass.testReferenceArrayIntrinsics()
testClass.testComileTimeUnknownArrayReceiver()
testStaticIntArrayIntrinsics()
testStaticLongArrayIntrinsics()
testStaticReferenceArrayIntrinsics()
return "OK"
}
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt
+3 -2
View File
@@ -37,7 +37,8 @@ import java.util.concurrent.ConcurrentHashMap
internal class NativeMapping : DefaultMapping() {
data class BridgeKey(val target: IrSimpleFunction, val bridgeDirections: BridgeDirections)
enum class AtomicFunctionType {
COMPARE_AND_EXCHANGE, COMPARE_AND_SET, GET_AND_SET, GET_AND_ADD;
COMPARE_AND_EXCHANGE, COMPARE_AND_SET, GET_AND_SET, GET_AND_ADD,
ATOMIC_GET_ARRAY_ELEMENT, ATOMIC_SET_ARRAY_ELEMENT, COMPARE_AND_EXCHANGE_ARRAY_ELEMENT, COMPARE_AND_SET_ARRAY_ELEMENT, GET_AND_SET_ARRAY_ELEMENT, GET_AND_ADD_ARRAY_ELEMENT;
}
data class AtomicFunctionKey(val field: IrField, val type: AtomicFunctionType)
@@ -124,4 +125,4 @@ internal class ContextLogger(val context: LoggingContext) {
internal fun LoggingContext.logMultiple(messageBuilder: ContextLogger.() -> Unit) {
if (!inVerbosePhase) return
with(ContextLogger(this)) { messageBuilder() }
}
}
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt
+5
View File
@@ -488,6 +488,11 @@ internal class CodegenLlvmHelpers(private val generationState: NativeGenerationS
val CompareAndSwapVolatileHeapRef by lazyRtFunction
val GetAndSetVolatileHeapRef by lazyRtFunction
// TODO: Consider implementing them directly in the code generator.
val Kotlin_arrayGetElementAddress by lazyRtFunction
val Kotlin_intArrayGetElementAddress by lazyRtFunction
val Kotlin_longArrayGetElementAddress by lazyRtFunction
val tlsMode by lazy {
when (target) {
KonanTarget.WASM32,
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt
+89 -51
View File
@@ -16,8 +16,7 @@ import org.jetbrains.kotlin.ir.declarations.IrField
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.expressions.*
import org.jetbrains.kotlin.ir.symbols.IrConstructorSymbol
import org.jetbrains.kotlin.ir.types.classOrNull
import org.jetbrains.kotlin.ir.types.getClass
import org.jetbrains.kotlin.ir.types.*
import org.jetbrains.kotlin.ir.util.findAnnotation
internal enum class IntrinsicType {
@@ -103,6 +102,13 @@ internal enum class IntrinsicType {
COMPARE_AND_EXCHANGE,
GET_AND_SET,
GET_AND_ADD,
// Atomic arrays
ATOMIC_GET_ARRAY_ELEMENT,
ATOMIC_SET_ARRAY_ELEMENT,
COMPARE_AND_EXCHANGE_ARRAY_ELEMENT,
COMPARE_AND_SET_ARRAY_ELEMENT,
GET_AND_SET_ARRAY_ELEMENT,
GET_AND_ADD_ARRAY_ELEMENT
}
internal enum class ConstantConstructorIntrinsicType {
@@ -261,6 +267,12 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
IntrinsicType.COMPARE_AND_EXCHANGE -> emitCompareAndSwap(callSite, args, resultSlot)
IntrinsicType.GET_AND_SET -> emitGetAndSet(callSite, args, resultSlot)
IntrinsicType.GET_AND_ADD -> emitGetAndAdd(callSite, args)
IntrinsicType.ATOMIC_GET_ARRAY_ELEMENT -> emitAtomicGetArrayElement(callSite, args, resultSlot)
IntrinsicType.ATOMIC_SET_ARRAY_ELEMENT -> emitAtomicSetArrayElement(callSite, args)
IntrinsicType.COMPARE_AND_EXCHANGE_ARRAY_ELEMENT -> emitCompareAndExchangeArrayElement(callSite, args, resultSlot)
IntrinsicType.COMPARE_AND_SET_ARRAY_ELEMENT -> emitCompareAndSetArrayElement(callSite, args)
IntrinsicType.GET_AND_SET_ARRAY_ELEMENT -> emitGetAndSetArrayElement(callSite, args, resultSlot)
IntrinsicType.GET_AND_ADD_ARRAY_ELEMENT -> emitGetAndAddArrayElement(callSite, args)
IntrinsicType.GET_CONTINUATION,
IntrinsicType.RETURN_IF_SUSPENDED,
IntrinsicType.INTEROP_BITS_TO_FLOAT,
@@ -322,32 +334,16 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
}
private fun FunctionGenerationContext.emitCmpExchange(callSite: IrCall, args: List<LLVMValueRef>, mode: CmpExchangeMode, resultSlot: LLVMValueRef?): LLVMValueRef {
val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!!
val address: LLVMValueRef
val expected: LLVMValueRef
val new: LLVMValueRef
if (callSite.dispatchReceiver != null) {
require(!field.isStatic)
require(args.size == 3)
address = environment.getObjectFieldPointer(args[0], field)
expected = args[1]
new = args[2]
} else {
require(field.isStatic)
require(args.size == 2)
address = environment.getStaticFieldPointer(field)
expected = args[0]
new = args[1]
}
require(args.size == 3) { "The call to ${callSite.symbol.owner.name.asString()} expects 3 value arguments." }
return if (isObjectRef(args[1])) {
require(context.memoryModel == MemoryModel.EXPERIMENTAL)
when (mode) {
CmpExchangeMode.SET -> call(llvm.CompareAndSetVolatileHeapRef, listOf(address, expected, new))
CmpExchangeMode.SWAP -> call(llvm.CompareAndSwapVolatileHeapRef, listOf(address, expected, new),
CmpExchangeMode.SET -> call(llvm.CompareAndSetVolatileHeapRef, args)
CmpExchangeMode.SWAP -> call(llvm.CompareAndSwapVolatileHeapRef, args,
environment.calculateLifetime(callSite), resultSlot = resultSlot)
}
} else {
val cmp = LLVMBuildAtomicCmpXchg(builder, address, expected, new,
val cmp = LLVMBuildAtomicCmpXchg(builder, args[0], args[1], args[2],
LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
SingleThread = 0
@@ -358,46 +354,88 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
}
private fun FunctionGenerationContext.emitAtomicRMW(callSite: IrCall, args: List<LLVMValueRef>, op: LLVMAtomicRMWBinOp, resultSlot: LLVMValueRef?): LLVMValueRef {
val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!!
val address: LLVMValueRef
val value: LLVMValueRef
if (callSite.dispatchReceiver != null) {
require(!field.isStatic)
require(args.size == 2)
address = environment.getObjectFieldPointer(args[0], field)
value = args[1]
} else {
require(field.isStatic)
require(args.size == 1)
address = environment.getStaticFieldPointer(field)
value = args[0]
}
return if (isObjectRef(value)) {
require(args.size == 2) { "The call to ${callSite.symbol.owner.name.asString()} expects 2 value arguments." }
return if (isObjectRef(args[1])) {
require(op == LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg)
require(context.memoryModel == MemoryModel.EXPERIMENTAL)
call(llvm.GetAndSetVolatileHeapRef, listOf(address, value),
call(llvm.GetAndSetVolatileHeapRef, args,
environment.calculateLifetime(callSite), resultSlot = resultSlot)
} else {
LLVMBuildAtomicRMW(builder, op, address, value,
LLVMBuildAtomicRMW(builder, op, args[0], args[1],
LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent,
singleThread = 0
)!!
}
}
private fun FunctionGenerationContext.emitCompareAndSet(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitCmpExchange(callSite, args, CmpExchangeMode.SET, null)
}
private fun FunctionGenerationContext.emitCompareAndSwap(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitCmpExchange(callSite, args, CmpExchangeMode.SWAP, resultSlot)
}
private fun FunctionGenerationContext.emitGetAndSet(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg, resultSlot)
}
private fun FunctionGenerationContext.emitGetAndAdd(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitAtomicRMW(callSite, args, LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpAdd, null)
private fun FunctionGenerationContext.transformArgsForVolatile(callSite: IrCall, args: List<LLVMValueRef>): List<LLVMValueRef> {
val field = context.mapping.functionToVolatileField[callSite.symbol.owner]!!
return if (callSite.dispatchReceiver != null) {
require(!field.isStatic)
listOf(environment.getObjectFieldPointer(args[0], field)) + args.drop(1)
} else {
require(field.isStatic)
listOf(environment.getStaticFieldPointer(field)) + args
}
}
private fun FunctionGenerationContext.emitCompareAndSet(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitCmpExchange(callSite, transformArgsForVolatile(callSite, args), CmpExchangeMode.SET, null)
}
private fun FunctionGenerationContext.emitCompareAndSwap(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitCmpExchange(callSite, transformArgsForVolatile(callSite, args), CmpExchangeMode.SWAP, resultSlot)
}
private fun FunctionGenerationContext.emitGetAndSet(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitAtomicRMW(callSite, transformArgsForVolatile(callSite, args), LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg, resultSlot)
}
private fun FunctionGenerationContext.emitGetAndAdd(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitAtomicRMW(callSite, transformArgsForVolatile(callSite, args), LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpAdd, null)
}
private fun FunctionGenerationContext.arrayGetElementAddress(callSite: IrCall, array: LLVMValueRef, index: LLVMValueRef): LLVMValueRef {
val receiver = callSite.extensionReceiver
require(receiver != null)
return when {
receiver.type.isIntArray() -> call(llvm.Kotlin_intArrayGetElementAddress, listOf(array, index))
receiver.type.isLongArray() -> call(llvm.Kotlin_longArrayGetElementAddress, listOf(array, index))
receiver.type.isArray() -> call(llvm.Kotlin_arrayGetElementAddress, listOf(array, index), environment.calculateLifetime(callSite))
else -> error("Only IntArray, LongArray and Array<T> are supported for atomic array intrinsics.")
}
}
private fun FunctionGenerationContext.emitAtomicSetArrayElement(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
require(args.size == 3) { "The call to ${callSite.symbol.owner.name.asString()} expects 3 value arguments." }
val address = arrayGetElementAddress(callSite, args[0], args[1])
storeAny(args[2], address, onStack = false, isVolatile = true)
return theUnitInstanceRef.llvm
}
private fun FunctionGenerationContext.emitAtomicGetArrayElement(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
require(args.size == 2) { "The call to ${callSite.symbol.owner.name.asString()} expects 2 value arguments." }
val address = arrayGetElementAddress(callSite, args[0], args[1])
return loadSlot(address, isVar = true, resultSlot, memoryOrder = LLVMAtomicOrdering.LLVMAtomicOrderingSequentiallyConsistent)
}
private fun FunctionGenerationContext.transformArgsForAtomicArray(callSite: IrCall, args: List<LLVMValueRef>): List<LLVMValueRef> {
val address = arrayGetElementAddress(callSite, args[0], args[1])
return listOf(address) + args.drop(2)
}
private fun FunctionGenerationContext.emitGetAndSetArrayElement(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitAtomicRMW(callSite, transformArgsForAtomicArray(callSite, args), LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpXchg, resultSlot)
}
private fun FunctionGenerationContext.emitGetAndAddArrayElement(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitAtomicRMW(callSite, transformArgsForAtomicArray(callSite, args), LLVMAtomicRMWBinOp.LLVMAtomicRMWBinOpAdd, null)
}
private fun FunctionGenerationContext.emitCompareAndExchangeArrayElement(callSite: IrCall, args: List<LLVMValueRef>, resultSlot: LLVMValueRef?): LLVMValueRef {
return emitCmpExchange(callSite, transformArgsForAtomicArray(callSite, args), CmpExchangeMode.SWAP, resultSlot)
}
private fun FunctionGenerationContext.emitCompareAndSetArrayElement(callSite: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
return emitCmpExchange(callSite, transformArgsForAtomicArray(callSite, args), CmpExchangeMode.SET, null)
}
private fun FunctionGenerationContext.emitGetNativeNullPtr(): LLVMValueRef =
llvm.kNullInt8Ptr
@@ -849,4 +887,4 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
llvm.doubleType -> llvm.float64(value.toDouble())
else -> context.reportCompilationError("Unexpected primitive type: $type")
}
}
}
kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp
+5
View File
@@ -9,6 +9,7 @@
#include "Runtime.h"
#include "Exceptions.h"
#include "MemorySharedRefs.hpp"
#include "Natives.h"
#define touchType(type) void touch##type(type*) {}
#define touchFunction(function) void* touch##function() { return reinterpret_cast<void*>(&::function); }
@@ -83,6 +84,10 @@ touchFunction(Kotlin_processArrayInMark)
touchFunction(Kotlin_processFieldInMark)
touchFunction(Kotlin_processEmptyObjectInMark)
touchFunction(Kotlin_arrayGetElementAddress)
touchFunction(Kotlin_intArrayGetElementAddress)
touchFunction(Kotlin_longArrayGetElementAddress)
#ifdef __cplusplus
} // extern "C"
#endif
kotlin-native/runtime/src/main/cpp/Natives.cpp
+19
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
#include <cinttypes>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
@@ -114,4 +115,22 @@ void Kotlin_CPointer_CopyMemory(KNativePtr to, KNativePtr from, KInt count) {
memcpy(to, from, count);
}
RUNTIME_NOTHROW RUNTIME_PURE KRef* Kotlin_arrayGetElementAddress(KRef array, KInt index) {
ArrayHeader* arr = array->array();
RuntimeAssert(index >= 0 && static_cast<uint32_t>(index) < arr->count_, "Index %" PRId32 " must be in [0, %" PRIu32 ")", index, arr->count_);
return ArrayAddressOfElementAt(arr, index);
}
RUNTIME_NOTHROW RUNTIME_PURE KInt* Kotlin_intArrayGetElementAddress(KRef array, KInt index) {
ArrayHeader* arr = array->array();
RuntimeAssert(index >= 0 && static_cast<uint32_t>(index) < arr->count_, "Index %" PRId32 " must be in [0, %" PRIu32 ")", index, arr->count_);
return IntArrayAddressOfElementAt(arr, index);
}
RUNTIME_NOTHROW RUNTIME_PURE KLong* Kotlin_longArrayGetElementAddress(KRef array, KInt index) {
ArrayHeader* arr = array->array();
RuntimeAssert(index >= 0 && static_cast<uint32_t>(index) < arr->count_, "Index %" PRId32 " must be in [0, %" PRIu32 ")", index, arr->count_);
return LongArrayAddressOfElementAt(arr, index);
}
} // extern "C"
kotlin-native/runtime/src/main/cpp/Natives.h
+11
View File
@@ -62,6 +62,14 @@ inline const KInt* IntArrayAddressOfElementAt(const ArrayHeader* obj, KInt index
return AddressOfElementAt<KInt>(obj, index);
}
inline KLong* LongArrayAddressOfElementAt(ArrayHeader* obj, KInt index) {
return AddressOfElementAt<KLong>(obj, index);
}
inline const KLong* LongArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) {
return AddressOfElementAt<KLong>(obj, index);
}
// Consider aligning of base to sizeof(T).
template <typename T>
inline T* PrimitiveArrayAddressOfElementAt(ArrayHeader* obj, KInt index) {
@@ -90,6 +98,9 @@ void Kotlin_io_Console_println0();
void Kotlin_io_Console_println0ToStdErr();
void Kotlin_NativePtrArray_set(KRef thiz, KInt index, KNativePtr value);
KNativePtr Kotlin_NativePtrArray_get(KConstRef thiz, KInt index);
RUNTIME_NOTHROW RUNTIME_PURE KRef* Kotlin_arrayGetElementAddress(KRef array, KInt index);
RUNTIME_NOTHROW RUNTIME_PURE KInt* Kotlin_intArrayGetElementAddress(KRef array, KInt index);
RUNTIME_NOTHROW RUNTIME_PURE KLong* Kotlin_longArrayGetElementAddress(KRef array, KInt index);
#ifdef __cplusplus
}
kotlin-native/runtime/src/main/kotlin/kotlin/concurrent/AtomicArrays.kt
+199
View File
@@ -0,0 +1,199 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package kotlin.concurrent
import kotlin.native.internal.*
import kotlin.reflect.*
import kotlin.concurrent.*
import kotlin.native.concurrent.*
/**
* Atomically gets the value of the [IntArray][this] element at the given [index].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_GET_ARRAY_ELEMENT)
internal external fun IntArray.atomicGet(index: Int): Int
/**
* Atomically sets the value of the [IntArray][this] element at the given [index] to the [new value][newValue].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_SET_ARRAY_ELEMENT)
internal external fun IntArray.atomicSet(index: Int, newValue: Int)
/**
* Atomically sets the value of the [IntArray][this] element at the given [index] to the [new value][newValue]
* and returns the old value of the element.
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.GET_AND_SET_ARRAY_ELEMENT)
internal external fun IntArray.getAndSet(index: Int, newValue: Int): Int
/**
* Atomically adds the [given value][delta] to the [IntArray][this] element at the given [index]
* and returns the old value of the element.
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.GET_AND_ADD_ARRAY_ELEMENT)
internal external fun IntArray.getAndAdd(index: Int, delta: Int): Int
/**
* Atomically sets the value of the [IntArray][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue] and returns the old value of the element in any case.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_EXCHANGE_ARRAY_ELEMENT)
internal external fun IntArray.compareAndExchange(index: Int, expectedValue: Int, newValue: Int): Int
/**
* Atomically sets the value of the [IntArray][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue].
* Returns true if the operation was successful and false only if the current value of the element was not equal to the expected value.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_SET_ARRAY_ELEMENT)
internal external fun IntArray.compareAndSet(index: Int, expectedValue: Int, newValue: Int): Boolean
/**
* Atomically gets the value of the [LongArray][this] element at the given [index].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_GET_ARRAY_ELEMENT)
internal external fun LongArray.atomicGet(index: Int): Long
/**
* Atomically sets the value of the [LongArray][this] element at the given [index] to the [new value][newValue].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_SET_ARRAY_ELEMENT)
internal external fun LongArray.atomicSet(index: Int, newValue: Long)
/**
* Atomically sets the value of the [LongArray][this] element at the given [index] to the [new value][newValue]
* and returns the old value of the element.
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.GET_AND_SET_ARRAY_ELEMENT)
internal external fun LongArray.getAndSet(index: Int, newValue: Long): Long
/**
* Atomically adds the [given value][delta] to the [LongArray][this] element at the given [index]
* and returns the old value of the element.
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.GET_AND_ADD_ARRAY_ELEMENT)
internal external fun LongArray.getAndAdd(index: Int, delta: Long): Long
/**
* Atomically sets the value of the [LongArray][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue] and returns the old value of the element in any case.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_EXCHANGE_ARRAY_ELEMENT)
internal external fun LongArray.compareAndExchange(index: Int, expectedValue: Long, newValue: Long): Long
/**
* Atomically sets the value of the [LongArray][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue].
* Returns true if the operation was successful and false only if the current value of the element was not equal to the expected value.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_SET_ARRAY_ELEMENT)
internal external fun LongArray.compareAndSet(index: Int, expectedValue: Long, newValue: Long): Boolean
/**
* Atomically gets the value of the [Array<T>][this] element at the given [index].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_GET_ARRAY_ELEMENT)
internal external fun <T> Array<T>.atomicGet(index: Int): T
/**
* Atomically sets the value of the [Array<T>][this] element at the given [index] to the [new value][newValue].
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.ATOMIC_SET_ARRAY_ELEMENT)
internal external fun <T> Array<T>.atomicSet(index: Int, newValue: T)
/**
* Atomically sets the value of the [Array<T>][this] element at the given [index] to the [new value][newValue]
* and returns the old value of the element.
*
* Provides sequential consistent ordering guarantees.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.GET_AND_SET_ARRAY_ELEMENT)
internal external fun <T> Array<T>.getAndSet(index: Int, value: T): T
/**
* Atomically sets the value of the [Array<T>][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue] and returns the old value of the element in any case.
*
* Comparison of values is done by reference.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_EXCHANGE_ARRAY_ELEMENT)
internal external fun <T> Array<T>.compareAndExchange(index: Int, expectedValue: T, newValue: T): T
/**
* Atomically sets the value of the [Array<T>][this] element at the given [index] to the [new value][newValue]
* if the current value equals the [expected value][expectedValue].
* Returns true if the operation was successful and false only if the current value of the element was not equal to the expected value.
*
* Comparison of values is done by reference.
*
* Provides sequential consistent ordering guarantees and never fails spuriously.
*
* NOTE: Ensure that the provided [index] does not exceed the size of the [array][this]. Exceeding the array size may result in undefined behavior.
*/
@TypedIntrinsic(IntrinsicType.COMPARE_AND_SET_ARRAY_ELEMENT)
internal external fun <T> Array<T>.compareAndSet(index: Int, expectedValue: T, newValue: T): Boolean
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt
+8 -1
View File
@@ -91,5 +91,12 @@ internal class IntrinsicType {
const val COMPARE_AND_EXCHANGE = "COMPARE_AND_EXCHANGE"
const val GET_AND_SET = "GET_AND_SET"
const val GET_AND_ADD = "GET_AND_ADD"
const val ATOMIC_GET_ARRAY_ELEMENT = "ATOMIC_GET_ARRAY_ELEMENT"
const val ATOMIC_SET_ARRAY_ELEMENT = "ATOMIC_SET_ARRAY_ELEMENT"
const val COMPARE_AND_EXCHANGE_ARRAY_ELEMENT = "COMPARE_AND_EXCHANGE_ARRAY_ELEMENT"
const val GET_AND_SET_ARRAY_ELEMENT = "GET_AND_SET_ARRAY_ELEMENT"
const val GET_AND_ADD_ARRAY_ELEMENT = "GET_AND_ADD_ARRAY_ELEMENT"
const val COMPARE_AND_SET_ARRAY_ELEMENT = "COMPARE_AND_SET_ARRAY_ELEMENT"
}
}
}
native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/FirNativeCodegenBoxTestGenerated.java
Generated
+6
View File
@@ -40127,6 +40127,12 @@ public class FirNativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTe
KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/volatile"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true);
}
@Test
@TestMetadata("atomicArrayIntrinsics.kt")
public void testAtomicArrayIntrinsics() throws Exception {
runTest("compiler/testData/codegen/box/volatile/atomicArrayIntrinsics.kt");
}
@Test
@TestMetadata("crossModuleIntrinsic.kt")
public void testCrossModuleIntrinsic() throws Exception {
native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/FirNativeCodegenBoxTestNoPLGenerated.java
Generated
+6
View File
@@ -41161,6 +41161,12 @@ public class FirNativeCodegenBoxTestNoPLGenerated extends AbstractNativeCodegenB
KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/volatile"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true);
}
@Test
@TestMetadata("atomicArrayIntrinsics.kt")
public void testAtomicArrayIntrinsics() throws Exception {
runTest("compiler/testData/codegen/box/volatile/atomicArrayIntrinsics.kt");
}
@Test
@TestMetadata("crossModuleIntrinsic.kt")
public void testCrossModuleIntrinsic() throws Exception {
native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java
Generated
+6
View File
@@ -39611,6 +39611,12 @@ public class NativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTest
KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/volatile"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true);
}
@Test
@TestMetadata("atomicArrayIntrinsics.kt")
public void testAtomicArrayIntrinsics() throws Exception {
runTest("compiler/testData/codegen/box/volatile/atomicArrayIntrinsics.kt");
}
@Test
@TestMetadata("crossModuleIntrinsic.kt")
public void testCrossModuleIntrinsic() throws Exception {
native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestNoPLGenerated.java
Generated
+6
View File
@@ -40128,6 +40128,12 @@ public class NativeCodegenBoxTestNoPLGenerated extends AbstractNativeCodegenBoxT
KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/volatile"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.NATIVE, true);
}
@Test
@TestMetadata("atomicArrayIntrinsics.kt")
public void testAtomicArrayIntrinsics() throws Exception {
runTest("compiler/testData/codegen/box/volatile/atomicArrayIntrinsics.kt");
}
@Test
@TestMetadata("crossModuleIntrinsic.kt")
public void testCrossModuleIntrinsic() throws Exception {