[K/N][New MM] Support thread state switching

Including

* Support thread state switching in codegen
* Introduce and use GCUnsafeCall annotation
* Switch thread state in C++ runtime code

Also

* Register current thread in Mark&Sweep tests
* Store MemoryState in Worker instance
* Set worker tid in WorkerInit
This commit is contained in:
Ilya Matveev
2021-01-25 19:06:27 +07:00
committed by Space
parent 0b46ed3cde
commit 4d346d3735
86 changed files with 1340 additions and 721 deletions
@@ -194,6 +194,7 @@ targetList.each { target ->
'-Xmulti-platform', '-Xopt-in=kotlin.RequiresOptIn', '-Xinline-classes',
'-Xopt-in=kotlin.contracts.ExperimentalContracts',
'-Xopt-in=kotlin.ExperimentalMultiplatform',
'-Xopt-in=kotlin.native.internal.InternalForKotlinNative',
'-Xallow-result-return-type',
*commonSrc.toList(),
*testAnnotationCommon.toList(),
@@ -32,4 +32,5 @@ object KonanFqNames {
val objCMethod = FqName("kotlinx.cinterop.ObjCMethod")
val hasFinalizer = FqName("kotlin.native.internal.HasFinalizer")
val hasFreezeHook = FqName("kotlin.native.internal.HasFreezeHook")
val gcUnsafeCall = FqName("kotlin.native.internal.GCUnsafeCall")
}
@@ -9,10 +9,7 @@ import org.jetbrains.kotlin.descriptors.Modality
import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
import org.jetbrains.kotlin.ir.IrStatement
import org.jetbrains.kotlin.ir.builders.*
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrValueParameter
import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.declarations.impl.IrFunctionImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrValueParameterImpl
import org.jetbrains.kotlin.ir.expressions.IrExpression
@@ -73,10 +70,11 @@ internal class KotlinBridgeBuilder(
cName: String,
stubs: KotlinStubs,
isExternal: Boolean,
foreignExceptionMode: ForeignExceptionMode.Mode
foreignExceptionMode: ForeignExceptionMode.Mode,
origin: IrDeclarationOrigin
) {
private var counter = 0
private val bridge: IrFunction = createKotlinBridge(startOffset, endOffset, cName, stubs, isExternal, foreignExceptionMode)
private val bridge: IrFunction = createKotlinBridge(startOffset, endOffset, cName, stubs, isExternal, foreignExceptionMode, origin)
val irBuilder: IrBuilderWithScope = irBuilder(stubs.irBuiltIns, bridge.symbol).at(startOffset, endOffset)
fun addParameter(type: IrType): IrValueParameter {
@@ -110,12 +108,13 @@ private fun createKotlinBridge(
cBridgeName: String,
stubs: KotlinStubs,
isExternal: Boolean,
foreignExceptionMode: ForeignExceptionMode.Mode
foreignExceptionMode: ForeignExceptionMode.Mode,
origin: IrDeclarationOrigin
): IrFunction {
val bridge = IrFunctionImpl(
startOffset,
endOffset,
IrDeclarationOrigin.DEFINED,
origin,
IrSimpleFunctionSymbolImpl(),
Name.identifier(cBridgeName),
DescriptorVisibilities.PRIVATE,
@@ -151,7 +150,9 @@ internal class KotlinCBridgeBuilder(
isKotlinToC: Boolean,
foreignExceptionMode: ForeignExceptionMode.Mode = ForeignExceptionMode.default
) {
private val kotlinBridgeBuilder = KotlinBridgeBuilder(startOffset, endOffset, cName, stubs, isExternal = isKotlinToC, foreignExceptionMode)
private val origin: CBridgeOrigin = if (isKotlinToC) CBridgeOrigin.KOTLIN_TO_C_BRIDGE else CBridgeOrigin.C_TO_KOTLIN_BRIDGE
private val kotlinBridgeBuilder = KotlinBridgeBuilder(startOffset, endOffset, cName, stubs, isExternal = isKotlinToC, foreignExceptionMode, origin)
private val cBridgeBuilder = CFunctionBuilder()
val kotlinIrBuilder: IrBuilderWithScope get() = kotlinBridgeBuilder.irBuilder
@@ -251,3 +252,8 @@ internal class CCallBuilder {
append(')')
}
}
sealed class CBridgeOrigin(name: String): IrDeclarationOriginImpl(name, isSynthetic = true) {
object KOTLIN_TO_C_BRIDGE: CBridgeOrigin("KOTLIN_TO_C_BRIDGE")
object C_TO_KOTLIN_BRIDGE: CBridgeOrigin("C_TO_KOTLIN_BRIDGE")
}
@@ -291,6 +291,8 @@ fun <T> IrConstructorCall.getAnnotationValueOrNull(name: String): T? {
fun IrFunction.externalSymbolOrThrow(): String? {
annotations.findAnnotation(RuntimeNames.symbolNameAnnotation)?.let { return it.getAnnotationStringValue() }
annotations.findAnnotation(KonanFqNames.gcUnsafeCall)?.let { return it.getAnnotationStringValue("callee") }
if (annotations.hasAnnotation(KonanFqNames.objCMethod)) return null
if (annotations.hasAnnotation(KonanFqNames.typedIntrinsic)) return null
@@ -299,7 +301,7 @@ fun IrFunction.externalSymbolOrThrow(): String? {
if (origin == InternalAbi.INTERNAL_ABI_ORIGIN) return null
throw Error("external function ${this.longName} must have @TypedIntrinsic, @SymbolName or @ObjCMethod annotation")
throw Error("external function ${this.longName} must have @TypedIntrinsic, @SymbolName, @GCUnsafeCall or @ObjCMethod annotation")
}
val IrFunction.isBuiltInOperator get() = origin == IrBuiltIns.BUILTIN_OPERATOR
@@ -9,6 +9,7 @@ package org.jetbrains.kotlin.backend.konan.llvm
import kotlinx.cinterop.*
import llvm.*
import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.cgen.CBridgeOrigin
import org.jetbrains.kotlin.backend.konan.descriptors.ClassGlobalHierarchyInfo
import org.jetbrains.kotlin.backend.konan.llvm.objc.*
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
@@ -16,6 +17,8 @@ import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.backend.konan.ir.*
import org.jetbrains.kotlin.backend.konan.llvm.ThreadState.Native
import org.jetbrains.kotlin.backend.konan.llvm.ThreadState.Runnable
import org.jetbrains.kotlin.descriptors.konan.CompiledKlibModuleOrigin
import org.jetbrains.kotlin.ir.expressions.IrDelegatingConstructorCall
import org.jetbrains.kotlin.ir.expressions.IrGetObjectValue
@@ -91,6 +94,10 @@ internal sealed class ExceptionHandler {
}
}
internal enum class ThreadState {
Native, Runnable
}
val LLVMValueRef.name:String?
get() = LLVMGetValueName(this)?.toKString()
@@ -341,7 +348,7 @@ internal class StackLocalsManagerImpl(
internal class FunctionGenerationContext(val function: LLVMValueRef,
val codegen: CodeGenerator,
startLocation: LocationInfo?,
endLocation: LocationInfo?,
private val endLocation: LocationInfo?,
internal val irFunction: IrFunction? = null): ContextUtils {
override val context = codegen.context
@@ -528,6 +535,19 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
//-------------------------------------------------------------------------//
fun switchThreadState(state: ThreadState) {
check(context.memoryModel == MemoryModel.EXPERIMENTAL) {
"Thread state switching is allowed in the experimental memory model only."
}
check(!forbidRuntime) {
"Attempt to switch the thread state when runtime is forbidden"
}
when (state) {
Native -> call(context.llvm.Kotlin_mm_switchThreadStateNative, emptyList())
Runnable -> call(context.llvm.Kotlin_mm_switchThreadStateRunnable, emptyList())
}.let {} // Force exhaustive.
}
fun call(llvmFunction: LLVMValueRef, args: List<LLVMValueRef>,
resultLifetime: Lifetime = Lifetime.IRRELEVANT,
exceptionHandler: ExceptionHandler = ExceptionHandler.None,
@@ -755,7 +775,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
return LLVMBuildExtractElement(builder, vector, index, name)!!
}
fun filteringExceptionHandler(codeContext: CodeContext, foreignExceptionMode: ForeignExceptionMode.Mode): ExceptionHandler {
fun filteringExceptionHandler(codeContext: CodeContext, foreignExceptionMode: ForeignExceptionMode.Mode, switchThreadState: Boolean): ExceptionHandler {
val lpBlock = basicBlockInFunction("filteringExceptionHandler", position()?.start)
val wrapExceptionMode = context.config.target.family.isAppleFamily &&
@@ -769,6 +789,10 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
}
LLVMAddClause(landingpad, LLVMConstNull(kInt8Ptr))
if (switchThreadState) {
switchThreadState(Runnable)
}
val fatalForeignExceptionBlock = basicBlock("fatalForeignException", position()?.start)
val forwardKotlinExceptionBlock = basicBlock("forwardKotlinException", position()?.start)
@@ -829,9 +853,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
}
}
fun kotlinExceptionHandler(
block: FunctionGenerationContext.(exception: LLVMValueRef) -> Unit
): ExceptionHandler {
fun kotlinExceptionHandler(block: FunctionGenerationContext.(exception: LLVMValueRef) -> Unit): ExceptionHandler {
val lpBlock = basicBlock("kotlinExceptionHandler", position()?.end)
appendingTo(lpBlock) {
@@ -1248,6 +1270,14 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
if (isObjectType(returnType!!)) {
returnSlot = LLVMGetParam(function, numParameters(function.type) - 1)
}
if (context.memoryModel == MemoryModel.EXPERIMENTAL &&
irFunction?.origin == CBridgeOrigin.C_TO_KOTLIN_BRIDGE) {
check(!forbidRuntime) { "Attempt to switch the thread state when runtime is forbidden" }
positionAtEnd(prologueBb)
switchThreadState(Runnable)
}
positionAtEnd(localsInitBb)
slotsPhi = phi(kObjHeaderPtrPtr)
// Is removed by DCE trivially, if not needed.
@@ -1307,8 +1337,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
returnType == voidType -> {
releaseVars()
assert(returnSlot == null)
if (!forbidRuntime && context.memoryModel == MemoryModel.EXPERIMENTAL)
call(context.llvm.Kotlin_mm_safePointFunctionEpilogue, emptyList())
handleEpilogueForExperimentalMM(context.llvm.Kotlin_mm_safePointFunctionEpilogue)
LLVMBuildRetVoid(builder)
}
returns.isNotEmpty() -> {
@@ -1318,8 +1347,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
updateReturnRef(returnPhi, returnSlot!!)
}
releaseVars()
if (!forbidRuntime && context.memoryModel == MemoryModel.EXPERIMENTAL)
call(context.llvm.Kotlin_mm_safePointFunctionEpilogue, emptyList())
handleEpilogueForExperimentalMM(context.llvm.Kotlin_mm_safePointFunctionEpilogue)
LLVMBuildRet(builder, returnPhi)
}
// Do nothing, all paths throw.
@@ -1360,8 +1388,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
}
releaseVars()
if (!forbidRuntime && context.memoryModel == MemoryModel.EXPERIMENTAL)
call(context.llvm.Kotlin_mm_safePointExceptionUnwind, emptyList())
handleEpilogueForExperimentalMM(context.llvm.Kotlin_mm_safePointExceptionUnwind)
LLVMBuildResume(builder, landingpad)
}
@@ -1371,6 +1398,18 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
slotsPhi = null
}
private fun handleEpilogueForExperimentalMM(safePointFunction: LLVMValueRef) {
if (context.memoryModel == MemoryModel.EXPERIMENTAL) {
if (!forbidRuntime) {
call(safePointFunction, emptyList())
}
if (irFunction?.origin == CBridgeOrigin.C_TO_KOTLIN_BRIDGE) {
check(!forbidRuntime) { "Generating a bridge when runtime is forbidden" }
switchThreadState(Native)
}
}
}
private val kotlinExceptionRtti: ConstPointer
get() = constPointer(importGlobal(
"_ZTI18ExceptionObjHolder", // typeinfo for ObjHolder
@@ -487,6 +487,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
val getObjCKotlinTypeInfo by lazy { importRtFunction("GetObjCKotlinTypeInfo") }
val missingInitImp by lazy { importRtFunction("MissingInitImp") }
val Kotlin_mm_switchThreadStateNative by lazy { importRtFunction("Kotlin_mm_switchThreadStateNative") }
val Kotlin_mm_switchThreadStateRunnable by lazy { importRtFunction("Kotlin_mm_switchThreadStateRunnable") }
val Kotlin_Interop_DoesObjectConformToProtocol by lazyRtFunction
val Kotlin_Interop_IsObjectKindOfClass by lazyRtFunction
@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.backend.common.ir.allParameters
import org.jetbrains.kotlin.backend.common.ir.allParametersCount
import org.jetbrains.kotlin.backend.common.lower.inline.InlinerExpressionLocationHint
import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.cgen.CBridgeOrigin
import org.jetbrains.kotlin.backend.konan.descriptors.*
import org.jetbrains.kotlin.backend.konan.ir.*
import org.jetbrains.kotlin.backend.konan.llvm.coverage.LLVMCoverageInstrumentation
@@ -673,7 +674,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
}
}
override val exceptionHandler get() = ExceptionHandler.Caller
override val exceptionHandler: ExceptionHandler
get() = ExceptionHandler.Caller
override fun genThrow(exception: LLVMValueRef) {
val objHeaderPtr = functionGenerationContext.bitcast(codegen.kObjHeaderPtr, exception)
@@ -1030,7 +1032,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
private inner abstract class CatchingScope : InnerScopeImpl() {
/**
* The LLVM `landingpad` such that if invoked function throws an exception,
* The LLVM `landingpad` such that if an invoked function throws an exception,
* then this exception is passed to [handler].
*/
private val landingpad: LLVMBasicBlockRef by lazy {
@@ -1087,9 +1089,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
}
}
override val exceptionHandler: ExceptionHandler get() = object : ExceptionHandler.Local() {
override val unwind get() = landingpad
}
override val exceptionHandler: ExceptionHandler
get() = object : ExceptionHandler.Local() {
override val unwind get() = landingpad
}
override fun genThrow(exception: LLVMValueRef) {
jumpToHandler(exception)
@@ -2332,17 +2335,39 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
//-------------------------------------------------------------------------//
private val IrFunction.needsNativeThreadState: Boolean
get() {
// We assume that call site thread state switching is required for interop calls only.
val result = context.memoryModel == MemoryModel.EXPERIMENTAL && origin == CBridgeOrigin.KOTLIN_TO_C_BRIDGE
if (result) {
check(isExternal)
check(!annotations.hasAnnotation(KonanFqNames.gcUnsafeCall))
check(annotations.hasAnnotation(RuntimeNames.filterExceptions))
}
return result
}
private fun call(function: IrFunction, llvmFunction: LLVMValueRef, args: List<LLVMValueRef>,
resultLifetime: Lifetime): LLVMValueRef {
check(!function.isTypedIntrinsic)
val needsNativeThreadState = function.needsNativeThreadState
val exceptionHandler = function.annotations.findAnnotation(RuntimeNames.filterExceptions)?.let {
val foreignExceptionMode = ForeignExceptionMode.byValue(it.getAnnotationValueOrNull<String>("mode"))
functionGenerationContext.filteringExceptionHandler(currentCodeContext, foreignExceptionMode)
functionGenerationContext.filteringExceptionHandler(currentCodeContext, foreignExceptionMode, needsNativeThreadState)
} ?: currentCodeContext.exceptionHandler
if (needsNativeThreadState) {
functionGenerationContext.switchThreadState(ThreadState.Native)
}
val result = call(llvmFunction, args, resultLifetime, exceptionHandler)
if (!function.isSuspend && function.returnType.isNothing()) {
functionGenerationContext.unreachable()
when {
!function.isSuspend && function.returnType.isNothing() ->
functionGenerationContext.unreachable()
needsNativeThreadState ->
functionGenerationContext.switchThreadState(ThreadState.Runnable)
}
if (LLVMGetReturnType(getFunctionType(llvmFunction)) == voidType) {
@@ -28,6 +28,10 @@ abstract class AbstractKonanIrMangler(private val withReturnType: Boolean) : IrB
// Treat any `@SymbolName` declaration as exported.
return true
}
if (annotations.hasAnnotation(KonanFqNames.gcUnsafeCall)) {
// Treat any `@GCUnsafeCall` declaration as exported.
return true
}
if (annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)) {
// Treat any `@ExportForCppRuntime` declaration as exported.
return true
@@ -99,6 +103,10 @@ abstract class AbstractKonanDescriptorMangler : DescriptorBasedKotlinManglerImpl
// Treat any `@SymbolName` declaration as exported.
return true
}
if (annotations.hasAnnotation(KonanFqNames.gcUnsafeCall)) {
// Treat any `@GCUnsafeCall` declaration as exported.
return true
}
if (annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)) {
// Treat any `@ExportForCppRuntime` declaration as exported.
return true
@@ -3830,6 +3830,11 @@ createInterop("cppSkia") {
it.defFile 'interop/cpp/skia.def'
}
createInterop("threadStates") {
it.defFile "interop/threadStates/threadStates.def"
it.extraOpts "-Xcompile-source", "$projectDir/interop/threadStates/threadStates.cpp"
}
if (PlatformInfo.isAppleTarget(project)) {
createInterop("objcSmoke") {
it.defFile 'interop/objc/objcSmoke.def'
@@ -4117,6 +4122,20 @@ interopTest("interop_callbacksAndVarargs") {
interop = 'ccallbacksAndVarargs'
}
interopTest("interop_threadStates") {
disabled = (project.testTarget == 'wasm32') || // No interop for wasm yet.
!isExperimentalMM // No thread state switching in the legacy MM.
source = "interop/threadStates/threadStates.kt"
interop = "threadStates"
}
interopTest("interop_threadStates_callbacksWithExceptions") {
disabled = (project.testTarget == 'wasm32') || // No interop for wasm yet.
!isExperimentalMM // No thread state switching in the legacy MM.
source = "interop/threadStates/callbacksWithExceptions.kt"
interop = "threadStates"
}
interopTest("interop_withSpaces") {
disabled = (project.testTarget == 'wasm32') // No interop for wasm yet.
interop ='withSpaces'
@@ -0,0 +1,95 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
import kotlin.native.internal.Debugging
import kotlin.test.*
import kotlinx.cinterop.staticCFunction
import threadStates.*
fun main() {
callbackWithException()
callbackWithFinally()
callbackWithFinallyNoCatch()
nestedCallbackWithException()
nestedCallbackWithFinally()
}
fun assertRunnableThreadState() {
assertTrue(Debugging.isThreadStateRunnable)
}
class CustomException() : Exception()
fun throwException() {
assertRunnableThreadState()
throw CustomException()
}
fun callbackWithException() {
try {
runCallback(staticCFunction(::throwException))
} catch (e: CustomException) {
assertRunnableThreadState()
return
} catch (e: Throwable) {
assertRunnableThreadState()
fail("Wrong exception type: ${e.message}")
}
fail("No exception thrown")
}
fun callbackWithFinally() {
try {
runCallback(staticCFunction(::throwException))
} catch (e: CustomException) {
assertRunnableThreadState()
return
} finally {
assertRunnableThreadState()
}
fail("No exception thrown")
}
fun callbackWithFinallyNoCatch() {
try {
try {
runCallback(staticCFunction(::throwException))
} finally {
assertRunnableThreadState()
}
assertRunnableThreadState()
} catch (_: CustomException) {}
}
fun nestedCallbackWithException() {
try {
runCallback(staticCFunction { ->
assertRunnableThreadState()
runCallback(staticCFunction(::throwException))
})
} catch (e: CustomException) {
assertRunnableThreadState()
return
} catch (e: Throwable) {
assertRunnableThreadState()
fail("Wrong exception type: ${e.message}")
}
fail("No exception thrown")
}
fun nestedCallbackWithFinally() {
try {
runCallback(staticCFunction { ->
assertRunnableThreadState()
runCallback(staticCFunction(::throwException))
})
} catch (e: CustomException) {
assertRunnableThreadState()
return
} finally {
assertRunnableThreadState()
}
fail("No exception thrown")
}
@@ -0,0 +1,20 @@
#include <thread>
#include <stdint.h>
#include <stdlib.h>
// Implemented in the runtime for test purposes.
extern "C" bool Kotlin_Debugging_isThreadStateNative();
extern "C" void assertNativeThreadState() {
if (!Kotlin_Debugging_isThreadStateNative()) {
printf("Incorrect thread state. Expected native thread state.");
abort();
}
}
extern "C" void runInNewThread(void(*callback)(void)) {
std::thread t([callback]() {
callback();
});
t.join();
}
@@ -0,0 +1,20 @@
language = C
---
#include <stdint.h>
void assertNativeThreadState();
void runCallback(void(*callback)(void)) {
assertNativeThreadState();
callback();
assertNativeThreadState();
}
int32_t answer() {
assertNativeThreadState();
return 42;
}
void runInNewThread(void(*callback)(void));
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
import kotlin.native.internal.Debugging
import kotlin.test.*
import kotlinx.cinterop.*
import threadStates.*
fun main() {
nativeCall()
callback()
nestedCalls()
directStaticCFunctionCall()
// TODO: Support runtime initialization for callbacks
// see: https://youtrack.jetbrains.com/issue/KT-44283
// callbackOnSeparateThread()
}
fun assertRunnableThreadState() {
assertTrue(Debugging.isThreadStateRunnable)
}
fun nativeCall() {
answer()
assertRunnableThreadState()
}
fun callback() {
runCallback(staticCFunction { ->
assertRunnableThreadState()
})
assertRunnableThreadState()
}
fun nestedCalls() {
runCallback(staticCFunction { ->
assertRunnableThreadState()
answer()
Unit
})
assertRunnableThreadState()
}
fun directStaticCFunctionCall() {
val funPtr = staticCFunction { ->
assertRunnableThreadState()
}
assertRunnableThreadState()
funPtr()
assertRunnableThreadState()
}
fun callbackOnSeparateThread() {
runInNewThread(staticCFunction { ->
assertRunnableThreadState()
})
}