diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt index 24f192c6187..bd348c83026 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt @@ -104,8 +104,6 @@ internal val cKeywords = setOf( "xor_eq" ) -private val cnameAnnotation = FqName("kotlin.native.CName") - private fun org.jetbrains.kotlin.types.KotlinType.isGeneric() = constructor.declarationDescriptor is TypeParameterDescriptor @@ -143,7 +141,7 @@ private fun AnnotationDescriptor.properValue(key: String) = private fun functionImplName(descriptor: DeclarationDescriptor, default: String, shortName: Boolean): String { assert(descriptor is FunctionDescriptor) - val annotation = descriptor.annotations.findAnnotation(cnameAnnotation) ?: return default + val annotation = descriptor.annotations.findAnnotation(RuntimeNames.cnameAnnotation) ?: return default val key = if (shortName) "shortName" else "externName" val value = annotation.properValue(key) return value.takeIf { value != null && value.isNotEmpty() } ?: default @@ -265,9 +263,10 @@ private class ExportedElement(val kind: ElementKind, val isFunction = declaration is FunctionDescriptor val isTopLevelFunction: Boolean get() { - if (declaration !is FunctionDescriptor || !declaration.annotations.hasAnnotation(cnameAnnotation)) + if (declaration !is FunctionDescriptor || + !declaration.annotations.hasAnnotation(RuntimeNames.cnameAnnotation)) return false - val annotation = declaration.annotations.findAnnotation(cnameAnnotation)!! + val annotation = declaration.annotations.findAnnotation(RuntimeNames.cnameAnnotation)!! val externName = annotation.properValue("externName") return externName != null && externName.isNotEmpty() } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 186c22b4b90..f5b66ef8296 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -128,6 +128,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val defaultNativeLibraries: List = mutableListOf().apply { add(if (debug) "debug.bc" else "release.bc") + add(if (memoryModel == MemoryModel.STRICT) "strict.bc" else "relaxed.bc") if (produce == CompilerOutputKind.PROGRAM) { addAll(distribution.launcherFiles) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/MemoryModel.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/MemoryModel.kt index 07fe13ca864..a618d5242c4 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/MemoryModel.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/MemoryModel.kt @@ -6,5 +6,5 @@ package org.jetbrains.kotlin.backend.konan enum class MemoryModel(val suffix: String) { STRICT("Strict"), - RELAXED("Relaxed)") + RELAXED("Relaxed") } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeNames.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeNames.kt index 82622dffc8b..6817e2965da 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeNames.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeNames.kt @@ -3,14 +3,18 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.name.FqName object RuntimeNames { - val symbolName = FqName("kotlin.native.SymbolName") + val symbolNameAnnotation = FqName("kotlin.native.SymbolName") + val cnameAnnotation = FqName("kotlin.native.CName") + val frozenAnnotation = FqName("kotlin.native.internal.Frozen") val exportForCppRuntime = FqName("kotlin.native.internal.ExportForCppRuntime") val exportForCompilerAnnotation = FqName("kotlin.native.internal.ExportForCompiler") val exportTypeInfoAnnotation = FqName("kotlin.native.internal.ExportTypeInfo") val cCall = FqName("kotlinx.cinterop.internal.CCall") + val objCMethodAnnotation = FqName("kotlinx.cinterop.ObjCMethod") val objCMethodImp = FqName("kotlinx.cinterop.ObjCMethodImp") val independent = FqName("kotlin.native.internal.Independent") val filterExceptions = FqName("kotlin.native.internal.FilterExceptions") val kotlinNativeInternalPackageName = FqName.fromSegments(listOf("kotlin", "native", "internal")) val associatedObjectKey = FqName("kotlin.reflect.AssociatedObjectKey") + val typedIntrinsicAnnotation = FqName("kotlin.native.internal.TypedIntrinsic") } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/LegacyDescriptorUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/LegacyDescriptorUtils.kt index 6e353ea4ea8..fbb497be5fe 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/LegacyDescriptorUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/LegacyDescriptorUtils.kt @@ -216,19 +216,14 @@ fun CallableMemberDescriptor.findSourceFile(): SourceFile { } } -internal val TypedIntrinsic = FqName("kotlin.native.internal.TypedIntrinsic") -private val symbolNameAnnotation = FqName("kotlin.native.SymbolName") -private val objCMethodAnnotation = FqName("kotlinx.cinterop.ObjCMethod") -private val frozenAnnotation = FqName("kotlin.native.internal.Frozen") - internal val DeclarationDescriptor.isFrozen: Boolean - get() = this.annotations.hasAnnotation(frozenAnnotation) || + get() = this.annotations.hasAnnotation(RuntimeNames.frozenAnnotation) || (this is org.jetbrains.kotlin.descriptors.ClassDescriptor // RTTI is used for non-reference type box or Objective-C object wrapper: && (!this.defaultType.binaryTypeIsReference() || this.isObjCClass())) internal val FunctionDescriptor.isTypedIntrinsic: Boolean - get() = this.annotations.hasAnnotation(TypedIntrinsic) + get() = this.annotations.hasAnnotation(RuntimeNames.typedIntrinsicAnnotation) // TODO: coalesce all our annotation value getters into fewer functions. fun getAnnotationValue(annotation: AnnotationDescriptor): String? { @@ -239,12 +234,12 @@ fun getAnnotationValue(annotation: AnnotationDescriptor): String? { } fun CallableMemberDescriptor.externalSymbolOrThrow(): String? { - this.annotations.findAnnotation(symbolNameAnnotation)?.let { + this.annotations.findAnnotation(RuntimeNames.symbolNameAnnotation)?.let { return getAnnotationValue(it)!! } - if (this.annotations.hasAnnotation(objCMethodAnnotation)) return null + if (this.annotations.hasAnnotation(RuntimeNames.objCMethodAnnotation)) return null - if (this.annotations.hasAnnotation(TypedIntrinsic)) return null + if (this.annotations.hasAnnotation(RuntimeNames.typedIntrinsicAnnotation)) return null if (this.annotations.hasAnnotation(RuntimeNames.cCall)) return null diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index 25fca1e401e..69f87aaf5e8 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -107,7 +107,7 @@ internal class KonanSymbols(context: Context, private val symbolTable: SymbolTab val arrayList = symbolTable.referenceClass(getArrayListClassDescriptor(context)) - val symbolName = topLevelClass(RuntimeNames.symbolName) + val symbolName = topLevelClass(RuntimeNames.symbolNameAnnotation) val filterExceptions = topLevelClass(RuntimeNames.filterExceptions) val exportForCppRuntime = topLevelClass(RuntimeNames.exportForCppRuntime) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt index 6e66f4e16ab..2ac16a852bb 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt @@ -16,20 +16,15 @@ import org.jetbrains.kotlin.backend.konan.ir.getObjCMethodInfo import org.jetbrains.kotlin.backend.konan.ir.isObjCClassMethod import org.jetbrains.kotlin.backend.konan.ir.isUnit import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler.isExported -import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol import org.jetbrains.kotlin.ir.types.* import org.jetbrains.kotlin.ir.util.fqNameForIrSerialization import org.jetbrains.kotlin.ir.types.getClass -import org.jetbrains.kotlin.ir.util.fqNameSafe import org.jetbrains.kotlin.ir.util.isSuspend import org.jetbrains.kotlin.ir.util.isVararg import org.jetbrains.kotlin.backend.konan.isInlinedNative import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.library.uniqueName -import org.jetbrains.kotlin.name.FqName // This file describes the ABI for Kotlin descriptors of exported declarations. @@ -53,34 +48,25 @@ object KonanMangler : KotlinManglerImpl() { override fun IrDeclaration.isPlatformSpecificExported(): Boolean { // TODO: revise val descriptorAnnotations = this.descriptor.annotations - if (descriptorAnnotations.hasAnnotation(symbolNameAnnotation)) { + if (descriptorAnnotations.hasAnnotation(RuntimeNames.symbolNameAnnotation)) { // Treat any `@SymbolName` declaration as exported. return true } - if (descriptorAnnotations.hasAnnotation(exportForCppRuntimeAnnotation)) { + if (descriptorAnnotations.hasAnnotation(RuntimeNames.exportForCppRuntime)) { // Treat any `@ExportForCppRuntime` declaration as exported. return true } - if (descriptorAnnotations.hasAnnotation(cnameAnnotation)) { + if (descriptorAnnotations.hasAnnotation(RuntimeNames.cnameAnnotation)) { // Treat `@CName` declaration as exported. return true } - if (descriptorAnnotations.hasAnnotation(exportForCompilerAnnotation)) { + if (descriptorAnnotations.hasAnnotation(RuntimeNames.exportForCompilerAnnotation)) { return true } return false } - private val symbolNameAnnotation = RuntimeNames.symbolName - - private val cnameAnnotation = FqName("kotlin.native.CName") - - private val exportForCppRuntimeAnnotation = RuntimeNames.exportForCppRuntime - - private val exportForCompilerAnnotation = RuntimeNames.exportForCompilerAnnotation - - override val IrFunction.argsPart get() = this.valueParameters.map { // TODO: there are clashes originating from ObjectiveC interop. @@ -128,7 +114,7 @@ object KonanMangler : KotlinManglerImpl() { } } - this.descriptor.annotations.findAnnotation(exportForCppRuntimeAnnotation)?.let { + this.descriptor.annotations.findAnnotation(RuntimeNames.exportForCppRuntime)?.let { val name = getAnnotationValue(it) ?: this.name.asString() return name // no wrapping currently required } @@ -148,8 +134,6 @@ internal val IrClass.writableTypeInfoSymbolName: String return "ktypew:" + this.fqNameForIrSerialization.toString() } -internal val theUnitInstanceName = "kobj:kotlin.Unit" - internal val IrClass.objectInstanceFieldSymbolName: String get() { assert (this.isExported()) @@ -194,21 +178,9 @@ internal fun RuntimeAware.getLlvmFunctionType(function: IrFunction): LLVMTypeRef return functionType(returnType, isVarArg = false, paramTypes = *paramTypes.toTypedArray()) } -internal fun RuntimeAware.getLlvmFunctionType(symbol: DataFlowIR.FunctionSymbol): LLVMTypeRef { - val returnType = if (symbol.returnsUnit) voidType else getLLVMType(symbol.returnParameter.type) - val paramTypes = ArrayList(symbol.parameters.map { getLLVMType(it.type) }) - if (isObjectType(returnType)) paramTypes.add(kObjHeaderPtrPtr) - - return functionType(returnType, isVarArg = false, paramTypes = *paramTypes.toTypedArray()) -} - internal val IrClass.typeInfoHasVtableAttached: Boolean get() = !this.isAbstract() && !this.isExternalObjCClass() -internal fun ModuleDescriptor.privateFunctionSymbolName(index: Int, functionName: String?) = "private_functions_${name.asString()}_${functionName}_$index" - -internal fun ModuleDescriptor.privateClassSymbolName(index: Int, className: String?) = "private_classes_${name.asString()}_${className}_$index" - internal val String.moduleConstructorName get() = "_Konan_init_${this}" diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index ca06daaea48..386901219e3 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -260,7 +260,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { throw IllegalArgumentException("function $name already exists") } - val externalFunction = LLVMGetNamedFunction(otherModule, name)!! + val externalFunction = LLVMGetNamedFunction(otherModule, name) ?: + throw Error("function $name not found") val functionType = getFunctionType(externalFunction) val function = LLVMAddFunction(llvmModule, name, functionType)!! @@ -418,13 +419,15 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { } private fun importRtFunction(name: String) = importFunction(name, runtime.llvmModule) + private fun importModelSpecificRtFunction(name: String) = + importRtFunction(name + context.memoryModel.suffix) private fun importRtGlobal(name: String) = importGlobal(name, runtime.llvmModule) - val allocInstanceFunction = importRtFunction("AllocInstance") - val allocArrayFunction = importRtFunction("AllocArrayInstance") - val initInstanceFunction = importRtFunction("InitInstance") - val initSharedInstanceFunction = importRtFunction("InitSharedInstance") + val allocInstanceFunction = importModelSpecificRtFunction("AllocInstance") + val allocArrayFunction = importModelSpecificRtFunction("AllocArrayInstance") + val initInstanceFunction = importModelSpecificRtFunction("InitInstance") + val initSharedInstanceFunction = importModelSpecificRtFunction("InitSharedInstance") val updateHeapRefFunction = importRtFunction("UpdateHeapRef") val updateStackRefFunction = importRtFunction("UpdateStackRef") val updateReturnRefFunction = importRtFunction("UpdateReturnRef") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt index f63a0b7bba4..56170441a7e 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt @@ -2,7 +2,7 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.cValuesOf import llvm.* -import org.jetbrains.kotlin.backend.konan.descriptors.TypedIntrinsic +import org.jetbrains.kotlin.backend.konan.RuntimeNames import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic import org.jetbrains.kotlin.backend.konan.llvm.objc.genObjCSelector import org.jetbrains.kotlin.backend.konan.reportCompilationError @@ -113,7 +113,7 @@ internal fun tryGetIntrinsicType(callSite: IrFunctionAccessExpression): Intrinsi private fun getIntrinsicType(callSite: IrFunctionAccessExpression): IntrinsicType { val function = callSite.symbol.owner - val annotation = function.descriptor.annotations.findAnnotation(TypedIntrinsic)!! + val annotation = function.descriptor.annotations.findAnnotation(RuntimeNames.typedIntrinsicAnnotation)!! val value = annotation.allValueArguments.getValue(Name.identifier("kind")).value as String return IntrinsicType.valueOf(value) } diff --git a/runtime/build.gradle b/runtime/build.gradle index 5f5b02d2810..1a8d6b27528 100644 --- a/runtime/build.gradle +++ b/runtime/build.gradle @@ -19,6 +19,8 @@ targetList.each { targetName -> dependsOn "${targetName}Launcher" dependsOn "${targetName}Debug" dependsOn "${targetName}Release" + dependsOn "${targetName}Strict" + dependsOn "${targetName}Relaxed" target targetName includeRuntime(delegate) linkerArgs project.file("../common/build/$targetName/hash.bc").path @@ -44,6 +46,20 @@ targetList.each { targetName -> target targetName includeRuntime(delegate) } + + task ("${targetName}Strict", type: CompileCppToBitcode) { + name "strict" + srcRoot file('src/strict') + target targetName + includeRuntime(delegate) + } + + task ("${targetName}Relaxed", type: CompileCppToBitcode) { + name "relaxed" + srcRoot file('src/relaxed') + target targetName + includeRuntime(delegate) + } } task hostRuntime(dependsOn: "${hostName}Runtime") diff --git a/runtime/src/launcher/cpp/androidLauncher.cpp b/runtime/src/launcher/cpp/androidLauncher.cpp index ba308d358b2..19062825c44 100644 --- a/runtime/src/launcher/cpp/androidLauncher.cpp +++ b/runtime/src/launcher/cpp/androidLauncher.cpp @@ -14,10 +14,11 @@ * limitations under the License. */ +#include "KString.h" #include "Memory.h" #include "Natives.h" +#include "Porting.h" #include "Runtime.h" -#include "KString.h" #include "Types.h" #ifdef KONAN_ANDROID @@ -212,7 +213,7 @@ extern "C" void RUNTIME_USED Konan_main( ANativeActivity* activity, void* savedState, size_t savedStateSize) { bool launchThread = activity->instance == nullptr; if (launchThread) { - launcherState = (LauncherState*)calloc(sizeof(LauncherState), 1); + launcherState = (LauncherState*)konan::calloc(sizeof(LauncherState), 1); launcherState->nativeActivityState = {activity, savedState, savedStateSize, nullptr}; activity->instance = launcherState; activity->callbacks->onDestroy = onDestroy; diff --git a/runtime/src/main/cpp/Arrays.cpp b/runtime/src/main/cpp/Arrays.cpp index 239cadfc236..cf3284392c6 100644 --- a/runtime/src/main/cpp/Arrays.cpp +++ b/runtime/src/main/cpp/Arrays.cpp @@ -443,8 +443,7 @@ OBJ_GETTER(Kotlin_CharArray_copyOf, KConstRef thiz, KInt newSize) { if (newSize < 0) { ThrowIllegalArgumentException(); } - ArrayHeader* result = AllocArrayInstance( - array->type_info(), newSize, OBJ_RESULT)->array(); + ArrayHeader* result = AllocArrayInstance(array->type_info(), newSize, OBJ_RESULT)->array(); KInt toCopy = array->count_ < newSize ? array->count_ : newSize; memcpy( PrimitiveArrayAddressOfElementAt(result, 0), @@ -606,8 +605,7 @@ OBJ_GETTER(Kotlin_ImmutableBlob_toByteArray, KConstRef thiz, KInt startIndex, KI ThrowArrayIndexOutOfBoundsException(); } KInt count = endIndex - startIndex; - ArrayHeader* result = AllocArrayInstance( - theByteArrayTypeInfo, count, OBJ_RESULT)->array(); + ArrayHeader* result = AllocArrayInstance(theByteArrayTypeInfo, count, OBJ_RESULT)->array(); memcpy(PrimitiveArrayAddressOfElementAt(result, 0), PrimitiveArrayAddressOfElementAt(array, startIndex), count); diff --git a/runtime/src/main/cpp/KString.cpp b/runtime/src/main/cpp/KString.cpp index c7425a7a22b..3cc413a0cab 100644 --- a/runtime/src/main/cpp/KString.cpp +++ b/runtime/src/main/cpp/KString.cpp @@ -728,6 +728,111 @@ void DisposeCString(char* cstring) { } // String.kt +OBJ_GETTER(Kotlin_String_replace, KString thiz, KChar oldChar, KChar newChar, KBoolean ignoreCase) { + auto count = thiz->count_; + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, count, OBJ_RESULT)->array(); + const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); + KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); + if (ignoreCase) { + KChar oldCharLower = towlower_Konan(oldChar); + for (KInt index = 0; index < count; ++index) { + KChar thizChar = *thizRaw++; + *resultRaw++ = towlower_Konan(thizChar) == oldCharLower ? newChar : thizChar; + } + } else { + for (KInt index = 0; index < count; ++index) { + KChar thizChar = *thizRaw++; + *resultRaw++ = thizChar == oldChar ? newChar : thizChar; + } + } + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_plusImpl, KString thiz, KString other) { + RuntimeAssert(thiz != nullptr, "this cannot be null"); + RuntimeAssert(other != nullptr, "other cannot be null"); + RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must be a string"); + RuntimeAssert(other->type_info() == theStringTypeInfo, "Must be a string"); + KInt result_length = thiz->count_ + other->count_; + if (result_length < thiz->count_ || result_length < other->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, result_length, OBJ_RESULT)->array(); + memcpy( + CharArrayAddressOfElementAt(result, 0), + CharArrayAddressOfElementAt(thiz, 0), + thiz->count_ * sizeof(KChar)); + memcpy( + CharArrayAddressOfElementAt(result, thiz->count_), + CharArrayAddressOfElementAt(other, 0), + other->count_ * sizeof(KChar)); + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_toUpperCase, KString thiz) { + auto count = thiz->count_; + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, count, OBJ_RESULT)->array(); + const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); + KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); + for (KInt index = 0; index < count; ++index) { + *resultRaw++ = towupper_Konan(*thizRaw++); + } + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_toLowerCase, KString thiz) { + auto count = thiz->count_; + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, count, OBJ_RESULT)->array(); + const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); + KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); + for (KInt index = 0; index < count; ++index) { + *resultRaw++ = towlower_Konan(*thizRaw++); + } + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) { + const ArrayHeader* array = thiz->array(); + RuntimeAssert(array->type_info() == theCharArrayTypeInfo, "Must use a char array"); + if (start < 0 || size < 0 || size > array->count_ - start) { + ThrowArrayIndexOutOfBoundsException(); + } + + if (size == 0) { + RETURN_RESULT_OF0(TheEmptyString); + } + + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, size, OBJ_RESULT)->array(); + memcpy(CharArrayAddressOfElementAt(result, 0), + CharArrayAddressOfElementAt(array, start), + size * sizeof(KChar)); + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_toCharArray, KString string, KInt start, KInt size) { + ArrayHeader* result = AllocArrayInstance(theCharArrayTypeInfo, size, OBJ_RESULT)->array(); + memcpy(CharArrayAddressOfElementAt(result, 0), + CharArrayAddressOfElementAt(string, start), + size * sizeof(KChar)); + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endIndex) { + if (startIndex < 0 || endIndex > thiz->count_ || startIndex > endIndex) { + // TODO: is it correct exception? + ThrowArrayIndexOutOfBoundsException(); + } + if (startIndex == endIndex) { + RETURN_RESULT_OF0(TheEmptyString); + } + KInt length = endIndex - startIndex; + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, length, OBJ_RESULT)->array(); + memcpy(CharArrayAddressOfElementAt(result, 0), + CharArrayAddressOfElementAt(thiz, startIndex), + length * sizeof(KChar)); + RETURN_OBJ(result->obj()); +} + KInt Kotlin_String_compareTo(KString thiz, KString other) { int result = memcmp( CharArrayAddressOfElementAt(thiz, 0), @@ -806,56 +911,6 @@ OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) { RETURN_RESULT_OF(utf16ToUtf8Impl, thiz, start, size); } -OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) { - const ArrayHeader* array = thiz->array(); - RuntimeAssert(array->type_info() == theCharArrayTypeInfo, "Must use a char array"); - if (start < 0 || size < 0 || size > array->count_ - start) { - ThrowArrayIndexOutOfBoundsException(); - } - - if (size == 0) { - RETURN_RESULT_OF0(TheEmptyString); - } - - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, size, OBJ_RESULT)->array(); - memcpy(CharArrayAddressOfElementAt(result, 0), - CharArrayAddressOfElementAt(array, start), - size * sizeof(KChar)); - RETURN_OBJ(result->obj()); -} - -OBJ_GETTER(Kotlin_String_toCharArray, KString string, KInt start, KInt size) { - ArrayHeader* result = AllocArrayInstance( - theCharArrayTypeInfo, size, OBJ_RESULT)->array(); - memcpy(CharArrayAddressOfElementAt(result, 0), - CharArrayAddressOfElementAt(string, start), - size * sizeof(KChar)); - RETURN_OBJ(result->obj()); -} - -OBJ_GETTER(Kotlin_String_plusImpl, KString thiz, KString other) { - RuntimeAssert(thiz != nullptr, "this cannot be null"); - RuntimeAssert(other != nullptr, "other cannot be null"); - RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must be a string"); - RuntimeAssert(other->type_info() == theStringTypeInfo, "Must be a string"); - KInt result_length = thiz->count_ + other->count_; - if (result_length < thiz->count_ || result_length < other->count_) { - ThrowArrayIndexOutOfBoundsException(); - } - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, result_length, OBJ_RESULT)->array(); - memcpy( - CharArrayAddressOfElementAt(result, 0), - CharArrayAddressOfElementAt(thiz, 0), - thiz->count_ * sizeof(KChar)); - memcpy( - CharArrayAddressOfElementAt(result, thiz->count_), - CharArrayAddressOfElementAt(other, 0), - other->count_ * sizeof(KChar)); - RETURN_OBJ(result->obj()); -} - KInt Kotlin_StringBuilder_insertString(KRef builder, KInt position, KString fromString) { auto toArray = builder->array(); RuntimeAssert(toArray->count_ >= fromString->count_ + position, "must be true"); @@ -907,52 +962,6 @@ KBoolean Kotlin_String_equalsIgnoreCase(KString thiz, KConstRef other) { return true; } -OBJ_GETTER(Kotlin_String_replace, KString thiz, KChar oldChar, KChar newChar, - KBoolean ignoreCase) { - auto count = thiz->count_; - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, count, OBJ_RESULT)->array(); - const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); - KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); - if (ignoreCase) { - KChar oldCharLower = towlower_Konan(oldChar); - for (KInt index = 0; index < count; ++index) { - KChar thizChar = *thizRaw++; - *resultRaw++ = towlower_Konan(thizChar) == oldCharLower ? newChar : thizChar; - } - } else { - for (KInt index = 0; index < count; ++index) { - KChar thizChar = *thizRaw++; - *resultRaw++ = thizChar == oldChar ? newChar : thizChar; - } - } - RETURN_OBJ(result->obj()); -} - -OBJ_GETTER(Kotlin_String_toUpperCase, KString thiz) { - auto count = thiz->count_; - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, count, OBJ_RESULT)->array(); - const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); - KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); - for (KInt index = 0; index < count; ++index) { - *resultRaw++ = towupper_Konan(*thizRaw++); - } - RETURN_OBJ(result->obj()); -} - -OBJ_GETTER(Kotlin_String_toLowerCase, KString thiz) { - auto count = thiz->count_; - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, count, OBJ_RESULT)->array(); - const KChar* thizRaw = CharArrayAddressOfElementAt(thiz, 0); - KChar* resultRaw = CharArrayAddressOfElementAt(result, 0); - for (KInt index = 0; index < count; ++index) { - *resultRaw++ = towlower_Konan(*thizRaw++); - } - RETURN_OBJ(result->obj()); -} - KBoolean Kotlin_String_regionMatches(KString thiz, KInt thizOffset, KString other, KInt otherOffset, KInt length, KBoolean ignoreCase) { @@ -1161,22 +1170,6 @@ KInt Kotlin_String_hashCode(KString thiz) { CharArrayAddressOfElementAt(thiz, 0), thiz->count_ * sizeof(KChar)); } -OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endIndex) { - if (startIndex < 0 || endIndex > thiz->count_ || startIndex > endIndex) { - // TODO: is it correct exception? - ThrowArrayIndexOutOfBoundsException(); - } - if (startIndex == endIndex) { - RETURN_RESULT_OF0(TheEmptyString); - } - KInt length = endIndex - startIndex; - ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, length, OBJ_RESULT)->array(); - memcpy(CharArrayAddressOfElementAt(result, 0), - CharArrayAddressOfElementAt(thiz, startIndex), - length * sizeof(KChar)); - RETURN_OBJ(result->obj()); -} - const KChar* Kotlin_String_utf16pointer(KString message) { RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string"); const KChar* utf16 = CharArrayAddressOfElementAt(message, 0); diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index bc5474b6750..2439e3e2fef 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -1525,6 +1525,648 @@ void garbageCollect() { #endif // USE_GC +void deinitInstanceBody(const TypeInfo* typeInfo, void* body) { + for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { + ObjHeader** location = reinterpret_cast( + reinterpret_cast(body) + typeInfo->objOffsets_[index]); + ZeroHeapRef(location); + } +} + +MemoryState* initMemory() { + RuntimeAssert(offsetof(ArrayHeader, typeInfoOrMeta_) + == + offsetof(ObjHeader, typeInfoOrMeta_), + "Layout mismatch"); + RuntimeAssert(offsetof(TypeInfo, typeInfo_) + == + offsetof(MetaObjHeader, typeInfo_), + "Layout mismatch"); + RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers") + RuntimeAssert(memoryState == nullptr, "memory state must be clear"); + memoryState = konanConstructInstance(); + INIT_EVENT(memoryState) +#if USE_GC + memoryState->toFree = konanConstructInstance(); + memoryState->roots = konanConstructInstance(); + memoryState->gcInProgress = false; + memoryState->gcSuspendCount = 0; + memoryState->toRelease = konanConstructInstance(); + initGcThreshold(memoryState, kGcThreshold); +#endif + atomicAdd(&aliveMemoryStatesCount, 1); + return memoryState; +} + +void deinitMemory(MemoryState* memoryState) { +#if USE_GC + do { + GC_LOG("Calling garbageCollect from DeinitMemory()\n") + garbageCollect(memoryState, true); + } while (memoryState->toRelease->size() > 0); + RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC"); + RuntimeAssert(memoryState->toRelease->size() == 0, "Some memory have not been released after GC"); + konanDestructInstance(memoryState->toFree); + konanDestructInstance(memoryState->roots); + konanDestructInstance(memoryState->toRelease); + RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty"); + RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty"); + +#endif // USE_GC + + bool lastMemoryState = atomicAdd(&aliveMemoryStatesCount, -1) == 0; + +#if TRACE_MEMORY + if (lastMemoryState && allocCount > 0) { + MEMORY_LOG("*** Memory leaks, leaked %d containers ***\n", allocCount); + dumpReachable("", memoryState->containers); + } +#else +#if USE_GC + if (lastMemoryState) + RuntimeAssert(allocCount == 0, "Memory leaks found"); +#endif +#endif + + PRINT_EVENT(memoryState) + DEINIT_EVENT(memoryState) + + konanFreeMemory(memoryState); + ::memoryState = nullptr; +} + +MemoryState* suspendMemory() { + auto result = ::memoryState; + ::memoryState = nullptr; + return result; +} + +void resumeMemory(MemoryState* state) { + RuntimeAssert(::memoryState == nullptr, "Cannot schedule on existing state"); + ::memoryState = state; +} + +void makeShareable(ContainerHeader* container) { + if (!container->frozen()) + container->makeShareable(); +} + +template +OBJ_GETTER(allocInstance, const TypeInfo* type_info) { + RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); + auto* state = memoryState; + auto container = ObjectContainer(state, type_info); +#if USE_GC + if (Strict) + rememberNewContainer(container.header()); + else + makeShareable(container.header()); +#endif // USE_GC + RETURN_OBJ(container.GetPlace()); +} + +template +OBJ_GETTER(allocArrayInstance, const TypeInfo* type_info, int32_t elements) { + RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); + if (elements < 0) ThrowIllegalArgumentException(); + auto* state = memoryState; + auto container = ArrayContainer(state, type_info, elements); +#if USE_GC + if (Strict) + rememberNewContainer(container.header()); + else + makeShareable(container.header()); +#endif // USE_GC + RETURN_OBJ(container.GetPlace()->obj()); +} + +template +OBJ_GETTER(initInstance, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + ObjHeader* value = *location; + if (value != nullptr) { + // OK'ish, inited by someone else. + RETURN_OBJ(value); + } + ObjHeader* object = allocInstance(typeInfo, OBJ_RESULT); + UpdateHeapRef(location, object); +#if KONAN_NO_EXCEPTIONS + ctor(object); + return object; +#else + try { + ctor(object); + return object; + } catch (...) { + UpdateReturnRef(OBJ_RESULT, nullptr); + ZeroHeapRef(location); + throw; + } +#endif +} + +template +OBJ_GETTER(initSharedInstance, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { +#if KONAN_NO_THREADS + ObjHeader* value = *location; + if (value != nullptr) { + // OK'ish, inited by someone else. + RETURN_OBJ(value); + } + ObjHeader* object = allocInstance(typeInfo, OBJ_RESULT); + UpdateHeapRef(location, object); +#if KONAN_NO_EXCEPTIONS + ctor(object); + FreezeSubgraph(object); + return object; +#else + try { + ctor(object); + if (Strict) + FreezeSubgraph(object); + return object; + } catch (...) { + UpdateReturnRef(OBJ_RESULT, nullptr); + ZeroHeapRef(location); + throw; + } +#endif // KONAN_NO_EXCEPTIONS +#else // KONAN_NO_THREADS + ObjHeader* value = *localLocation; + if (value != nullptr) RETURN_OBJ(value); + + ObjHeader* initializing = reinterpret_cast(1); + + // Spin lock. + while ((value = __sync_val_compare_and_swap(location, nullptr, initializing)) == initializing); + if (value != nullptr) { + // OK'ish, inited by someone else. + RETURN_OBJ(value); + } + ObjHeader* object = allocInstance(typeInfo, OBJ_RESULT); + RuntimeAssert(object->container()->normal() , "Shared object cannot be co-allocated"); + UpdateHeapRef(localLocation, object); +#if KONAN_NO_EXCEPTIONS + ctor(object); + if (Strict) + FreezeSubgraph(object); + UpdateHeapRef(location, object); + synchronize(); + return object; +#else // KONAN_NO_EXCEPTIONS + try { + ctor(object); + if (Strict) + FreezeSubgraph(object); + UpdateHeapRef(location, object); + synchronize(); + return object; + } catch (...) { + UpdateReturnRef(OBJ_RESULT, nullptr); + ZeroHeapRef(location); + ZeroHeapRef(localLocation); + synchronize(); + throw; + } +#endif // KONAN_NO_EXCEPTIONS +#endif // KONAN_NO_THREADS +} + +void setStackRef(ObjHeader** location, const ObjHeader* object) { + MEMORY_LOG("SetStackRef *%p: %p\n", location, object) + UPDATE_REF_EVENT(memoryState, nullptr, object, location, 1); + *const_cast(location) = object; +} + +void setHeapRef(ObjHeader** location, const ObjHeader* object) { + MEMORY_LOG("SetHeapRef *%p: %p\n", location, object) + UPDATE_REF_EVENT(memoryState, nullptr, object, location, 0); + if (object != nullptr) + addHeapRef(const_cast(object)); + *const_cast(location) = object; +} + +void zeroHeapRef(ObjHeader** location) { + MEMORY_LOG("ZeroHeapRef %p\n", location) + auto* value = *location; + if (value != nullptr) { + UPDATE_REF_EVENT(memoryState, value, nullptr, location, 0); + *location = nullptr; + releaseHeapRef(value); + } +} + +void zeroStackRef(ObjHeader** location) { + MEMORY_LOG("ZeroStackRef %p\n", location) +#if TRACE_MEMORY + auto* value = *location; + if (value != nullptr) { + UPDATE_REF_EVENT(memoryState, value, nullptr, location, 1); + *location = nullptr; + } +#else + *location = nullptr; +#endif +} + +void updateStackRef(ObjHeader** location, const ObjHeader* object) { + UPDATE_REF_EVENT(memoryState, *location, object, location, 1) + RuntimeAssert(object != reinterpret_cast(1), "Markers disallowed here"); + *const_cast(location) = object; +} + +void updateHeapRef(ObjHeader** location, const ObjHeader* object) { + UPDATE_REF_EVENT(memoryState, *location, object, location, 0); + ObjHeader* old = *location; + if (old != object) { + if (object != nullptr) { + addHeapRef(object); + } + *const_cast(location) = object; + if (reinterpret_cast(old) > 1) { + releaseHeapRef(old); + } + } +} + +void updateReturnRef(ObjHeader** returnSlot, const ObjHeader* value) { + UpdateStackRef(returnSlot, value); +} + +void updateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { + if (object != nullptr) { +#if KONAN_NO_THREADS + ObjHeader* old = *location; + if (old == nullptr) { + addHeapRef(const_cast(object)); + *const_cast(location) = object; + } +#else + addHeapRef(const_cast(object)); + auto old = __sync_val_compare_and_swap(location, nullptr, const_cast(object)); + if (old != nullptr) { + // Failed to store, was not null. + releaseHeapRef(const_cast(object)); + } +#endif + UPDATE_REF_EVENT(memoryState, old, object, location, 0); + } +} + +OBJ_GETTER(swapHeapRefLocked, + ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) { + lock(spinlock); + ObjHeader* oldValue = *location; + bool shallRelease = false; + // We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock. + if (oldValue == expectedValue) { + SetHeapRef(location, newValue); + shallRelease = oldValue != nullptr; + } + unlock(spinlock); + if (shallRelease) { + releaseHeapRef(oldValue); + } + // No need to rememberNewContainer(), as oldValue is already + // present on this worker. + UpdateReturnRef(OBJ_RESULT, oldValue); + return oldValue; +} + +void setHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) { + lock(spinlock); + ObjHeader* oldValue = *location; + // We do not use UpdateRef() here to avoid having ReleaseRef() on old value under the lock. + SetHeapRef(location, newValue); + unlock(spinlock); + if (oldValue != nullptr) + releaseHeapRef(oldValue); +} + +OBJ_GETTER(readHeapRefLocked, ObjHeader** location, int32_t* spinlock) { + MEMORY_LOG("ReadHeapRefLocked: %p\n", location) + lock(spinlock); + ObjHeader* value = *location; + auto* container = value ? value->container() : nullptr; + if (container != nullptr) + incrementRC(container); + unlock(spinlock); + if (container != nullptr) + enqueueDecrementRC(container); + RETURN_OBJ(value); +} + +OBJ_GETTER(readHeapRefNoLock, ObjHeader* object, KInt index) { + MEMORY_LOG("ReadHeapRefNoLock: %p index %d\n", object, index) + ObjHeader** location = reinterpret_cast( + reinterpret_cast(object) + object->type_info()->objOffsets_[index]); + ObjHeader* value = *location; +#if USE_GC + if (value != nullptr) + rememberNewContainer(value->container()); +#endif // USE_GC + RETURN_OBJ(value); +} + +void enterFrame(ObjHeader** start, int parameters, int count) { + MEMORY_LOG("EnterFrame %p: %d parameters %d locals\n", start, parameters, count) + FrameOverlay* frame = reinterpret_cast(start); + frame->previous = currentFrame; + currentFrame = frame; + // TODO: maybe compress in single value somehow. + frame->parameters = parameters; + frame->count = count; +} + +void leaveFrame(ObjHeader** start, int parameters, int count) { + MEMORY_LOG("LeaveFrame %p: %d parameters %d locals\n", start, parameters, count) + FrameOverlay* frame = reinterpret_cast(start); + currentFrame = frame->previous; +} + +void suspendGC() { + GC_LOG("suspendGC\n") + memoryState->gcSuspendCount++; +} + +void resumeGC() { + GC_LOG("resumeGC\n") + MemoryState* state = memoryState; + if (state->gcSuspendCount > 0) { + state->gcSuspendCount--; + if (state->toRelease != nullptr && + state->toRelease->size() >= state->gcThreshold && + state->gcSuspendCount == 0) { + + garbageCollect(state, false); + } + } +} + +void stopGC() { + GC_LOG("stopGC\n") + if (memoryState->toRelease != nullptr) { + memoryState->gcSuspendCount = 0; + garbageCollect(memoryState, true); + konanDestructInstance(memoryState->toRelease); + konanDestructInstance(memoryState->toFree); + konanDestructInstance(memoryState->roots); + memoryState->toRelease = nullptr; + memoryState->toFree = nullptr; + memoryState->roots = nullptr; + } +} + +void startGC() { + GC_LOG("startGC\n") + if (memoryState->toFree == nullptr) { + memoryState->toFree = konanConstructInstance(); + memoryState->toRelease = konanConstructInstance(); + memoryState->roots = konanConstructInstance(); + memoryState->gcSuspendCount = 0; + } +} + +void setGCThreshold(KInt value) { + GC_LOG("Kotlin_native_internal_setThreshold %d\n", value) + if (value > 0) { + initGcThreshold(memoryState, value); + } +} + +KInt getGCThreshold() { + GC_LOG("Kotlin_native_internal_getThreshold %d\n") + return memoryState->gcThreshold; +} + +KNativePtr createStablePointer(KRef any) { + if (any == nullptr) return nullptr; + MEMORY_LOG("CreateStablePointer for %p rc=%d\n", any, any->container() ? any->container()->refCount() : 0) + addHeapRef(any); + return reinterpret_cast(any); +} + +void disposeStablePointer(KNativePtr pointer) { + if (pointer == nullptr) return; + KRef ref = reinterpret_cast(pointer); + releaseHeapRef(ref); +} + +OBJ_GETTER(derefStablePointer, KNativePtr pointer) { + KRef ref = reinterpret_cast(pointer); +#if USE_GC + if (ref != nullptr) + rememberNewContainer(ref->container()); +#endif // USE_GC + RETURN_OBJ(ref); +} + +OBJ_GETTER(adoptStablePointer, KNativePtr pointer) { + synchronize(); + KRef ref = reinterpret_cast(pointer); + MEMORY_LOG("adopting stable pointer %p, rc=%d\n", \ + ref, (ref && ref->container()) ? ref->container()->refCount() : -1) + UpdateReturnRef(OBJ_RESULT, ref); + DisposeStablePointer(pointer); + return ref; +} + +bool clearSubgraphReferences(ObjHeader* root, bool checked) { +#if USE_GC + MEMORY_LOG("ClearSubgraphReferences %p\n", root) + if (root == nullptr) return true; + auto state = memoryState; + auto* container = root->container(); + + if (Shareable(container)) + // We assume, that frozen/shareable objects can be safely passed and not present + // in the GC candidate list. + // TODO: assert for that? + return true; + + ContainerHeaderSet visited; + if (!checked) { + hasExternalRefs(container, &visited); + } else { + // Now decrement RC of elements in toRelease set for reachibility analysis. + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto released = *it; + if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { + released->decRefCount(); + } + } + container->decRefCount(); + markGray(container); + auto bad = hasExternalRefs(container, &visited); + scanBlack(container); + // Restore original RC. + container->incRefCount(); + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto released = *it; + if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { + released->incRefCount(); + } + } + if (bad) { + return false; + } + } + + // Remove all no longer owned containers from GC structures. + // TODO: not very efficient traversal. + for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { + auto container = *it; + if (visited.count(container) != 0) { + MEMORY_LOG("removing %p from the toFree list\n", container) + container->resetBuffered(); + container->setColorAssertIfGreen(CONTAINER_TAG_GC_BLACK); + *it = markAsRemoved(container); + } + } + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto container = *it; + if (!isMarkedAsRemoved(container) && visited.count(container) != 0) { + MEMORY_LOG("removing %p from the toRelease list\n", container) + container->decRefCount(); + *it = markAsRemoved(container); + } + } + +#if TRACE_MEMORY + // Forget transferred containers. + for (auto* it: visited) { + state->containers->erase(it); + } +#endif + +#endif // USE_GC + return true; +} + +/** + * Theory of operations. + * + * Kotlin/Native supports object graph freezing, allowing to make certain subgraph immutable and thus + * suitable for safe sharing amongst multiple concurrent executors. This operation recursively operates + * on all objects reachable from the given object, and marks them as frozen. In frozen state object's + * fields cannot be modified, and so, lifetime of frozen objects correlates. Practically, it means + * that lifetimes of all strongly connected components are fully controlled by incoming reference + * counters, and so if we place all members of strongly connected component to the single container + * it could be correctly released by just atomic decrement on reference counter, without additional + * cycle collector run. + * So during subgraph freezing operation, we perform the following steps: + * - run Kosoraju-Sharir algorithm to find strongly connected components + * - put all objects in each strongly connected component into an artificial container + * (we assume that they all were in single element containers initially), single-object + * components remain in the same container + * - artificial container sums up outer reference counters of all its objects (i.e. + * incoming references from the same strongly connected component are not counted) + * - mark all object's headers as frozen + * + * Further reference counting on frozen objects is performed with atomic operations, and so frozen + * references could be passed across multiple threads. + */ +void freezeSubgraph(ObjHeader* root) { + if (root == nullptr) return; + // First check that passed object graph has no cycles. + // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir. + ContainerHeader* rootContainer = root->container(); + if (Shareable(rootContainer)) return; + + MEMORY_LOG("Freeze subgraph of %p\n", root) + + // Do DFS cycle detection. + bool hasCycles = false; + KRef firstBlocker = root->has_meta_object() && ((root->meta_object()->flags_ & MF_NEVER_FROZEN) != 0) ? + root : nullptr; + KStdVector order; + depthFirstTraversal(rootContainer, &hasCycles, &firstBlocker, &order); + if (firstBlocker != nullptr) { + MEMORY_LOG("See freeze blocker for %p: %p\n", root, firstBlocker) + ThrowFreezingException(root, firstBlocker); + } + ContainerHeaderSet newlyFrozen; + // Now unmark all marked objects, and freeze them, if no cycles detected. + if (hasCycles) { + freezeCyclic(rootContainer, order, &newlyFrozen); + } else { + freezeAcyclic(rootContainer, &newlyFrozen); + } + MEMORY_LOG("Graph of %p is %s with %d elements\n", root, hasCycles ? "cyclic" : "acyclic", newlyFrozen.size()) + +#if USE_GC + // Now remove frozen objects from the toFree list. + // TODO: optimize it by keeping ignored (i.e. freshly frozen) objects in the set, + // and use it when analyzing toFree during collection. + auto state = memoryState; + for (auto& container : *(state->toFree)) { + if (!isMarkedAsRemoved(container) && container->frozen()) { + RuntimeAssert(newlyFrozen.count(container) != 0, "Must be newly frozen"); + container = markAsRemoved(container); + } + } +#endif +} + +void ensureNeverFrozen(ObjHeader* object) { + auto* container = object->container(); + if (container == nullptr || container->frozen()) + ThrowFreezingException(object, object); + // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency, + // although there's subtle race with case, where other thread freezes the same object after check. + object->meta_object()->flags_ |= MF_NEVER_FROZEN; +} + +KBoolean ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) { + RuntimeAssert(where->container() != nullptr && where->container()->frozen(), "Must be used on frozen objects only"); + RuntimeAssert(what == nullptr || PermanentOrFrozen(what), + "Must be used with an immutable value"); + if (what != nullptr) { + // Now we check that `where` is not reachable from `what`. + // As we cannot modify objects while traversing, instead we remember all seen objects in a set. + KStdUnorderedSet seen; + KStdDeque queue; + if (what->container() != nullptr) + queue.push_back(what->container()); + bool acyclic = true; + while (!queue.empty() && acyclic) { + ContainerHeader* current = queue.front(); + queue.pop_front(); + seen.insert(current); + if (isAggregatingFrozenContainer(current)) { + ContainerHeader** subContainer = reinterpret_cast(current + 1); + for (int i = 0; i < current->objectCount(); ++i) { + if (seen.count(*subContainer) == 0) + queue.push_back(*subContainer++); + } + } else { + traverseContainerReferredObjects(current, [where, &queue, &acyclic, &seen](ObjHeader* obj) { + if (obj == where) { + acyclic = false; + } else { + auto* objContainer = obj->container(); + if (objContainer != nullptr && seen.count(objContainer) == 0) + queue.push_back(objContainer); + } + }); + } + } + if (!acyclic) return false; + } + UpdateHeapRef(reinterpret_cast( + reinterpret_cast(where) + where->type_info()->objOffsets_[index]), what); + // Fence on updated location? + return true; +} + +void shareAny(ObjHeader* obj) { + auto* container = obj->container(); + if (Shareable(container)) return; + RuntimeCheck(container->objectCount() == 1, "Must be a single object container"); + container->makeShareable(); +} + } // namespace MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { @@ -1689,6 +2331,8 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun return result; } + +// Public API of the memory manager. extern "C" { void AddRefFromAssociatedObject(const ObjHeader* object) { @@ -1700,654 +2344,204 @@ void ReleaseRefFromAssociatedObject(const ObjHeader* object) { } void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) { - for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { - ObjHeader** location = reinterpret_cast( - reinterpret_cast(body) + typeInfo->objOffsets_[index]); - ZeroHeapRef(location); - } + deinitInstanceBody(typeInfo, body); } MemoryState* InitMemory() { - RuntimeAssert(offsetof(ArrayHeader, typeInfoOrMeta_) - == - offsetof(ObjHeader, typeInfoOrMeta_), - "Layout mismatch"); - RuntimeAssert(offsetof(TypeInfo, typeInfo_) - == - offsetof(MetaObjHeader, typeInfo_), - "Layout mismatch"); - RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers") - RuntimeAssert(memoryState == nullptr, "memory state must be clear"); - memoryState = konanConstructInstance(); - INIT_EVENT(memoryState) -#if USE_GC - memoryState->toFree = konanConstructInstance(); - memoryState->roots = konanConstructInstance(); - memoryState->gcInProgress = false; - memoryState->gcSuspendCount = 0; - memoryState->toRelease = konanConstructInstance(); - initGcThreshold(memoryState, kGcThreshold); -#endif - atomicAdd(&aliveMemoryStatesCount, 1); - return memoryState; + return initMemory(); } void DeinitMemory(MemoryState* memoryState) { -#if USE_GC - do { - GC_LOG("Calling garbageCollect from DeinitMemory()\n") - garbageCollect(memoryState, true); - } while (memoryState->toRelease->size() > 0); - RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC"); - RuntimeAssert(memoryState->toRelease->size() == 0, "Some memory have not been released after GC"); - konanDestructInstance(memoryState->toFree); - konanDestructInstance(memoryState->roots); - konanDestructInstance(memoryState->toRelease); - RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty"); - RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty"); - -#endif // USE_GC - - bool lastMemoryState = atomicAdd(&aliveMemoryStatesCount, -1) == 0; - -#if TRACE_MEMORY - if (lastMemoryState && allocCount > 0) { - MEMORY_LOG("*** Memory leaks, leaked %d containers ***\n", allocCount); - dumpReachable("", memoryState->containers); - } -#else -#if USE_GC - if (lastMemoryState) - RuntimeAssert(allocCount == 0, "Memory leaks found"); -#endif -#endif - - PRINT_EVENT(memoryState) - DEINIT_EVENT(memoryState) - - konanFreeMemory(memoryState); - ::memoryState = nullptr; + deinitMemory(memoryState); } MemoryState* SuspendMemory() { - auto result = ::memoryState; - ::memoryState = nullptr; - return result; + return suspendMemory(); } void ResumeMemory(MemoryState* state) { - RuntimeAssert(::memoryState == nullptr, "Cannot schedule on existing state"); - ::memoryState = state; + resumeMemory(state); } -OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { - RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - auto* state = memoryState; - auto container = ObjectContainer(state, type_info); -#if USE_GC - rememberNewContainer(container.header()); -#endif // USE_GC - RETURN_OBJ(container.GetPlace()); +OBJ_GETTER(AllocInstanceStrict, const TypeInfo* type_info) { + RETURN_RESULT_OF(allocInstance, type_info); } -OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements) { - RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); - if (elements < 0) ThrowIllegalArgumentException(); - auto* state = memoryState; - auto container = ArrayContainer(state, type_info, elements); -#if USE_GC - rememberNewContainer(container.header()); -#endif // USE_GC - RETURN_OBJ(container.GetPlace()->obj()); +OBJ_GETTER(AllocInstanceRelaxed, const TypeInfo* type_info) { + RETURN_RESULT_OF(allocInstance, type_info); } -OBJ_GETTER(InitInstance, - ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) { - ObjHeader* value = *location; - if (value != nullptr) { - // OK'ish, inited by someone else. - RETURN_OBJ(value); - } - ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); - UpdateHeapRef(location, object); -#if KONAN_NO_EXCEPTIONS - ctor(object); - return object; -#else - try { - ctor(object); - return object; - } catch (...) { - UpdateReturnRef(OBJ_RESULT, nullptr); - ZeroHeapRef(location); - throw; - } -#endif +OBJ_GETTER(AllocArrayInstanceStrict, const TypeInfo* typeInfo, int32_t elements) { + RETURN_RESULT_OF(allocArrayInstance, typeInfo, elements); } -OBJ_GETTER(InitSharedInstance, - ObjHeader** location, ObjHeader** localLocation, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) { -#if KONAN_NO_THREADS - ObjHeader* value = *location; - if (value != nullptr) { - // OK'ish, inited by someone else. - RETURN_OBJ(value); - } - ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); - UpdateHeapRef(location, object); -#if KONAN_NO_EXCEPTIONS - ctor(object); - FreezeSubgraph(object); - return object; -#else - try { - ctor(object); - FreezeSubgraph(object); - return object; - } catch (...) { - UpdateReturnRef(OBJ_RESULT, nullptr); - ZeroHeapRef(location); - throw; - } -#endif -#else - ObjHeader* value = *localLocation; - if (value != nullptr) RETURN_OBJ(value); +OBJ_GETTER(AllocArrayInstanceRelaxed, const TypeInfo* typeInfo, int32_t elements) { + RETURN_RESULT_OF(allocArrayInstance, typeInfo, elements); +} - ObjHeader* initializing = reinterpret_cast(1); +OBJ_GETTER(InitInstanceStrict, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initInstance, location, typeInfo, ctor); +} - // Spin lock. - while ((value = __sync_val_compare_and_swap(location, nullptr, initializing)) == initializing); - if (value != nullptr) { - // OK'ish, inited by someone else. - RETURN_OBJ(value); - } - ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); - RuntimeAssert(object->container()->normal() , "Shared object cannot be co-allocated"); - UpdateHeapRef(localLocation, object); -#if KONAN_NO_EXCEPTIONS - ctor(object); - FreezeSubgraph(object); - UpdateHeapRef(location, object); - synchronize(); - return object; -#else - try { - ctor(object); - FreezeSubgraph(object); - UpdateHeapRef(location, object); - synchronize(); - return object; - } catch (...) { - UpdateReturnRef(OBJ_RESULT, nullptr); - ZeroHeapRef(location); - ZeroHeapRef(localLocation); - synchronize(); - throw; - } -#endif -#endif +OBJ_GETTER(InitInstanceRelaxed, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initInstance, location, typeInfo, ctor); +} + +OBJ_GETTER(InitSharedInstanceStrict, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initSharedInstance, location, localLocation, typeInfo, ctor); +} + +OBJ_GETTER(InitSharedInstanceRelaxed, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(initSharedInstance, location, localLocation, typeInfo, ctor); } void SetStackRef(ObjHeader** location, const ObjHeader* object) { - MEMORY_LOG("SetStackRef *%p: %p\n", location, object) - UPDATE_REF_EVENT(memoryState, nullptr, object, location, 1); - *const_cast(location) = object; + setStackRef(location, object); } void SetHeapRef(ObjHeader** location, const ObjHeader* object) { - MEMORY_LOG("SetHeapRef *%p: %p\n", location, object) - UPDATE_REF_EVENT(memoryState, nullptr, object, location, 0); - if (object != nullptr) - addHeapRef(const_cast(object)); - *const_cast(location) = object; + setHeapRef(location, object); } void ZeroHeapRef(ObjHeader** location) { - MEMORY_LOG("ZeroHeapRef %p\n", location) - auto* value = *location; - if (value != nullptr) { - UPDATE_REF_EVENT(memoryState, value, nullptr, location, 0); - *location = nullptr; - releaseHeapRef(value); - } + zeroHeapRef(location); } void ZeroStackRef(ObjHeader** location) { - MEMORY_LOG("ZeroStackRef %p\n", location) -#if TRACE_MEMORY - auto* value = *location; - if (value != nullptr) { - UPDATE_REF_EVENT(memoryState, value, nullptr, location, 1); - *location = nullptr; - } -#else - *location = nullptr; -#endif + zeroStackRef(location); } void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { - UPDATE_REF_EVENT(memoryState, *location, object, location, 1) - RuntimeAssert(object != reinterpret_cast(1), "Markers disallowed here"); - *const_cast(location) = object; + updateStackRef(location, object); } void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { - UPDATE_REF_EVENT(memoryState, *location, object, location, 0); - ObjHeader* old = *location; - if (old != object) { - if (object != nullptr) { - addHeapRef(object); - } - *const_cast(location) = object; - if (reinterpret_cast(old) > 1) { - releaseHeapRef(old); - } - } + updateHeapRef(location, object); } void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* value) { - UpdateStackRef(returnSlot, value); + updateReturnRef(returnSlot, value); } void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { - if (object != nullptr) { -#if KONAN_NO_THREADS - ObjHeader* old = *location; - if (old == nullptr) { - addHeapRef(const_cast(object)); - *const_cast(location) = object; - } -#else - addHeapRef(const_cast(object)); - auto old = __sync_val_compare_and_swap(location, nullptr, const_cast(object)); - if (old != nullptr) { - // Failed to store, was not null. - releaseHeapRef(const_cast(object)); - } -#endif - UPDATE_REF_EVENT(memoryState, old, object, location, 0); - } + updateHeapRefIfNull(location, object); +} + +OBJ_GETTER(SwapHeapRefLocked, + ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) { + RETURN_RESULT_OF(swapHeapRefLocked, location, expectedValue, newValue, spinlock); +} + +void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) { + setHeapRefLocked(location, newValue, spinlock); +} + +OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock) { + RETURN_RESULT_OF(readHeapRefLocked, location, spinlock); +} + +OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, KInt index) { + RETURN_RESULT_OF(readHeapRefNoLock, object, index); } void EnterFrame(ObjHeader** start, int parameters, int count) { - MEMORY_LOG("EnterFrame %p: %d parameters %d locals\n", start, parameters, count) - FrameOverlay* frame = reinterpret_cast(start); - frame->previous = currentFrame; - currentFrame = frame; - // TODO: maybe compress in single value somehow. - frame->parameters = parameters; - frame->count = count; + enterFrame(start, parameters, count); } void LeaveFrame(ObjHeader** start, int parameters, int count) { - MEMORY_LOG("LeaveFrame %p: %d parameters %d locals\n", start, parameters, count) - FrameOverlay* frame = reinterpret_cast(start); - currentFrame = frame->previous; + leaveFrame(start, parameters, count); } void Kotlin_native_internal_GC_collect(KRef) { #if USE_GC - GC_LOG("Kotlin_native_internal_GC_collect\n") garbageCollect(); #endif } void Kotlin_native_internal_GC_suspend(KRef) { #if USE_GC - GC_LOG("Kotlin_native_internal_GC_suspend\n") - memoryState->gcSuspendCount++; + suspendGC(); #endif } void Kotlin_native_internal_GC_resume(KRef) { #if USE_GC - MemoryState* state = memoryState; - if (state->gcSuspendCount > 0) { - state->gcSuspendCount--; - if (state->toRelease != nullptr && - state->toRelease->size() >= state->gcThreshold && - state->gcSuspendCount == 0) { - GC_LOG("Kotlin_native_internal_GC_resume\n") - garbageCollect(state, false); - } - } + resumeGC(); #endif } void Kotlin_native_internal_GC_stop(KRef) { #if USE_GC - GC_LOG("Kotlin_native_internal_GC_stop\n") - if (memoryState->toRelease != nullptr) { - memoryState->gcSuspendCount = 0; - garbageCollect(memoryState, true); - konanDestructInstance(memoryState->toRelease); - konanDestructInstance(memoryState->toFree); - konanDestructInstance(memoryState->roots); - memoryState->toRelease = nullptr; - memoryState->toFree = nullptr; - memoryState->roots = nullptr; - } + stopGC(); #endif } void Kotlin_native_internal_GC_start(KRef) { #if USE_GC - GC_LOG("Kotlin_native_internal_GC_start\n") - if (memoryState->toFree == nullptr) { - memoryState->toFree = konanConstructInstance(); - memoryState->toRelease = konanConstructInstance(); - memoryState->roots = konanConstructInstance(); - memoryState->gcSuspendCount = 0; - } + startGC(); #endif } void Kotlin_native_internal_GC_setThreshold(KRef, KInt value) { #if USE_GC - GC_LOG("Kotlin_native_internal_setThreshold %d\n", value) - if (value > 0) { - initGcThreshold(memoryState, value); - } + setGCThreshold(value); #endif } KInt Kotlin_native_internal_GC_getThreshold(KRef) { #if USE_GC - GC_LOG("Kotlin_native_internal_getThreshold %d\n") - return memoryState->gcThreshold; + return getGCThreshold(); #else return -1; #endif } KNativePtr CreateStablePointer(KRef any) { - if (any == nullptr) return nullptr; - MEMORY_LOG("CreateStablePointer for %p rc=%d\n", any, any->container() ? any->container()->refCount() : 0) - addHeapRef(any); - return reinterpret_cast(any); + return createStablePointer(any); } void DisposeStablePointer(KNativePtr pointer) { - if (pointer == nullptr) return; - KRef ref = reinterpret_cast(pointer); - releaseHeapRef(ref); + disposeStablePointer(pointer); } OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { - KRef ref = reinterpret_cast(pointer); -#if USE_GC - if (ref != nullptr) - rememberNewContainer(ref->container()); -#endif // USE_GC - RETURN_OBJ(ref); + RETURN_RESULT_OF(derefStablePointer, pointer); } OBJ_GETTER(AdoptStablePointer, KNativePtr pointer) { - synchronize(); - KRef ref = reinterpret_cast(pointer); - MEMORY_LOG("adopting stable pointer %p, rc=%d\n", \ - ref, (ref && ref->container()) ? ref->container()->refCount() : -1) - UpdateReturnRef(OBJ_RESULT, ref); - DisposeStablePointer(pointer); - return ref; + RETURN_RESULT_OF(adoptStablePointer, pointer); } bool ClearSubgraphReferences(ObjHeader* root, bool checked) { -#if USE_GC - MEMORY_LOG("ClearSubgraphReferences %p\n", root) - if (root == nullptr) return true; - auto state = memoryState; - auto* container = root->container(); - - if (Shareable(container)) - // We assume, that frozen/shareable objects can be safely passed and not present - // in the GC candidate list. - // TODO: assert for that? - return true; - - ContainerHeaderSet visited; - if (!checked) { - hasExternalRefs(container, &visited); - } else { - // Now decrement RC of elements in toRelease set for reachibility analysis. - for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { - auto released = *it; - if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { - released->decRefCount(); - } - } - container->decRefCount(); - markGray(container); - auto bad = hasExternalRefs(container, &visited); - scanBlack(container); - // Restore original RC. - container->incRefCount(); - for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { - auto released = *it; - if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { - released->incRefCount(); - } - } - if (bad) { - return false; - } - } - - // Remove all no longer owned containers from GC structures. - // TODO: not very efficient traversal. - for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { - auto container = *it; - if (visited.count(container) != 0) { - MEMORY_LOG("removing %p from the toFree list\n", container) - container->resetBuffered(); - container->setColorAssertIfGreen(CONTAINER_TAG_GC_BLACK); - *it = markAsRemoved(container); - } - } - for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { - auto container = *it; - if (!isMarkedAsRemoved(container) && visited.count(container) != 0) { - MEMORY_LOG("removing %p from the toRelease list\n", container) - container->decRefCount(); - *it = markAsRemoved(container); - } - } - -#if TRACE_MEMORY - // Forget transferred containers. - for (auto* it: visited) { - state->containers->erase(it); - } -#endif - -#endif // USE_GC - return true; + return clearSubgraphReferences(root, checked); } -/** - * Theory of operations. - * - * Kotlin/Native supports object graph freezing, allowing to make certain subgraph immutable and thus - * suitable for safe sharing amongst multiple concurrent executors. This operation recursively operates - * on all objects reachable from the given object, and marks them as frozen. In frozen state object's - * fields cannot be modified, and so, lifetime of frozen objects correlates. Practically, it means - * that lifetimes of all strongly connected components are fully controlled by incoming reference - * counters, and so if we place all members of strongly connected component to the single container - * it could be correctly released by just atomic decrement on reference counter, without additional - * cycle collector run. - * So during subgraph freezing operation, we perform the following steps: - * - run Kosoraju-Sharir algorithm to find strongly connected components - * - put all objects in each strongly connected component into an artificial container - * (we assume that they all were in single element containers initially), single-object - * components remain in the same container - * - artificial container sums up outer reference counters of all its objects (i.e. - * incoming references from the same strongly connected component are not counted) - * - mark all object's headers as frozen - * - * Further reference counting on frozen objects is performed with atomic operations, and so frozen - * references could be passed across multiple threads. - */ void FreezeSubgraph(ObjHeader* root) { - if (root == nullptr) return; - // First check that passed object graph has no cycles. - // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir. - ContainerHeader* rootContainer = root->container(); - if (Shareable(rootContainer)) return; - - MEMORY_LOG("Freeze subgraph of %p\n", root) - - // Do DFS cycle detection. - bool hasCycles = false; - KRef firstBlocker = root->has_meta_object() && ((root->meta_object()->flags_ & MF_NEVER_FROZEN) != 0) ? - root : nullptr; - KStdVector order; - depthFirstTraversal(rootContainer, &hasCycles, &firstBlocker, &order); - if (firstBlocker != nullptr) { - MEMORY_LOG("See freeze blocker for %p: %p\n", root, firstBlocker) - ThrowFreezingException(root, firstBlocker); - } - ContainerHeaderSet newlyFrozen; - // Now unmark all marked objects, and freeze them, if no cycles detected. - if (hasCycles) { - freezeCyclic(rootContainer, order, &newlyFrozen); - } else { - freezeAcyclic(rootContainer, &newlyFrozen); - } - MEMORY_LOG("Graph of %p is %s with %d elements\n", root, hasCycles ? "cyclic" : "acyclic", newlyFrozen.size()) - -#if USE_GC - // Now remove frozen objects from the toFree list. - // TODO: optimize it by keeping ignored (i.e. freshly frozen) objects in the set, - // and use it when analyzing toFree during collection. - auto state = memoryState; - for (auto& container : *(state->toFree)) { - if (!isMarkedAsRemoved(container) && container->frozen()) { - RuntimeAssert(newlyFrozen.count(container) != 0, "Must be newly frozen"); - container = markAsRemoved(container); - } - } -#endif + freezeSubgraph(root); } // This function is called from field mutators to check if object's header is frozen. -// If object is frozen, an exception is thrown. +// If object is frozen or permanent, an exception is thrown. void MutationCheck(ObjHeader* obj) { auto* container = obj->container(); - if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj); -} - -OBJ_GETTER(SwapHeapRefLocked, - ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) { - lock(spinlock); - ObjHeader* oldValue = *location; - bool shallRelease = false; - // We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock. - if (oldValue == expectedValue) { - SetHeapRef(location, newValue); - shallRelease = oldValue != nullptr; - } - unlock(spinlock); - if (shallRelease) { - releaseHeapRef(oldValue); - } - // No need to rememberNewContainer(), as oldValue is already - // present on this worker. - UpdateReturnRef(OBJ_RESULT, oldValue); - return oldValue; -} - -void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) { - lock(spinlock); - ObjHeader* oldValue = *location; - // We do not use UpdateRef() here to avoid having ReleaseRef() on old value under the lock. - SetHeapRef(location, newValue); - unlock(spinlock); - if (oldValue != nullptr) - releaseHeapRef(oldValue); -} - -OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock) { - MEMORY_LOG("ReadHeapRefLocked: %p\n", location) - lock(spinlock); - ObjHeader* value = *location; - auto* container = value ? value->container() : nullptr; - if (container != nullptr) - incrementRC(container); - unlock(spinlock); - if (container != nullptr) - enqueueDecrementRC(container); - RETURN_OBJ(value); -} - -OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, KInt index) { - MEMORY_LOG("ReadHeapRefNoLock: %p index %d\n", object, index) - ObjHeader** location = reinterpret_cast( - reinterpret_cast(object) + object->type_info()->objOffsets_[index]); - ObjHeader* value = *location; -#if USE_GC - if (value != nullptr) - rememberNewContainer(value->container()); -#endif // USE_GC - RETURN_OBJ(value); + if (container == nullptr || container->frozen()) + ThrowInvalidMutabilityException(obj); } void EnsureNeverFrozen(ObjHeader* object) { - auto* container = object->container(); - if (container == nullptr || container->frozen()) - ThrowFreezingException(object, object); - // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency, - // although there's subtle race with case, where other thread freezes the same object after check. - object->meta_object()->flags_ |= MF_NEVER_FROZEN; + ensureNeverFrozen(object); } KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) { - RuntimeAssert(where->container() != nullptr && where->container()->frozen(), "Must be used on frozen objects only"); - RuntimeAssert(what == nullptr || PermanentOrFrozen(what), - "Must be used with an immutable value"); - if (what != nullptr) { - // Now we check that `where` is not reachable from `what`. - // As we cannot modify objects while traversing, instead we remember all seen objects in a set. - KStdUnorderedSet seen; - KStdDeque queue; - if (what->container() != nullptr) - queue.push_back(what->container()); - bool acyclic = true; - while (!queue.empty() && acyclic) { - ContainerHeader* current = queue.front(); - queue.pop_front(); - seen.insert(current); - if (isAggregatingFrozenContainer(current)) { - ContainerHeader** subContainer = reinterpret_cast(current + 1); - for (int i = 0; i < current->objectCount(); ++i) { - if (seen.count(*subContainer) == 0) - queue.push_back(*subContainer++); - } - } else { - traverseContainerReferredObjects(current, [where, &queue, &acyclic, &seen](ObjHeader* obj) { - if (obj == where) { - acyclic = false; - } else { - auto* objContainer = obj->container(); - if (objContainer != nullptr && seen.count(objContainer) == 0) - queue.push_back(objContainer); - } - }); - } - } - if (!acyclic) return false; - } - UpdateHeapRef(reinterpret_cast( - reinterpret_cast(where) + where->type_info()->objOffsets_[index]), what); - // Fence on updated location? - return true; + return ensureAcyclicAndSet(where, index, what); } void Kotlin_Any_share(ObjHeader* obj) { - auto* container = obj->container(); - if (Shareable(container)) return; - RuntimeCheck(container->objectCount() == 1, "Must be a single object container"); - container->makeShareable(); + shareAny(obj); } } // extern "C" diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 9b4ae005893..2eff73a89e5 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -426,11 +426,30 @@ void ResumeMemory(MemoryState* state); // Escape analysis algorithm is the provider of information for decision on exact aux slot // selection, and comes from upper bound esteemation of object lifetime. // +OBJ_GETTER(AllocInstanceStrict, const TypeInfo* type_info) RUNTIME_NOTHROW; +OBJ_GETTER(AllocInstanceRelaxed, const TypeInfo* type_info) RUNTIME_NOTHROW; OBJ_GETTER(AllocInstance, const TypeInfo* type_info) RUNTIME_NOTHROW; + +OBJ_GETTER(AllocArrayInstanceStrict, const TypeInfo* type_info, int32_t elements); +OBJ_GETTER(AllocArrayInstanceRelaxed, const TypeInfo* type_info, int32_t elements); OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements); + +OBJ_GETTER(InitInstanceStrict, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); +OBJ_GETTER(InitInstanceRelaxed, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); +OBJ_GETTER(InitInstance, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + +OBJ_GETTER(InitSharedInstanceStrict, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); +OBJ_GETTER(InitSharedInstanceRelaxed, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); +OBJ_GETTER(InitSharedInstance, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + +// Cleanup references inside object. void DeinitInstanceBody(const TypeInfo* typeInfo, void* body); -OBJ_GETTER(InitInstance, ObjHeader** location, const TypeInfo* type_info, - void (*ctor)(ObjHeader*)); // Weak reference operations. // Atomically clears counter object reference. diff --git a/runtime/src/main/cpp/Natives.h b/runtime/src/main/cpp/Natives.h index 467613c1755..3b8dddad3ca 100644 --- a/runtime/src/main/cpp/Natives.h +++ b/runtime/src/main/cpp/Natives.h @@ -84,72 +84,10 @@ inline const KRef* ArrayAddressOfElementAt(const ArrayHeader* obj, KInt index) { extern "C" { #endif -// RuntimeUtils.kt. OBJ_GETTER0(TheEmptyString); - -// Any.kt -KBoolean Kotlin_Any_equals(KConstRef thiz, KConstRef other); -KInt Kotlin_Any_hashCode(KConstRef thiz); -OBJ_GETTER(Kotlin_Any_toString, KConstRef thiz); - -// Arrays.kt -// TODO: those must be compiler intrinsics afterwards. -OBJ_GETTER(Kotlin_Array_clone, KConstRef thiz); -OBJ_GETTER(Kotlin_Array_get, KConstRef thiz, KInt index); -void Kotlin_Array_set(KRef thiz, KInt index, KConstRef value); -KInt Kotlin_Array_getArrayLength(KConstRef thiz); - -OBJ_GETTER(Kotlin_ByteArray_clone, KConstRef thiz); -KByte Kotlin_ByteArray_get(KConstRef thiz, KInt index); -void Kotlin_ByteArray_set(KRef thiz, KInt index, KByte value); -KInt Kotlin_ByteArray_getArrayLength(KConstRef thiz); - -OBJ_GETTER(Kotlin_CharArray_clone, KConstRef thiz); -KChar Kotlin_CharArray_get(KConstRef thiz, KInt index); -void Kotlin_CharArray_set(KRef thiz, KInt index, KChar value); -KInt Kotlin_CharArray_getArrayLength(KConstRef thiz); - -OBJ_GETTER(Kotlin_IntArray_clone, KConstRef thiz); -KInt Kotlin_IntArray_get(KConstRef thiz, KInt index); -void Kotlin_IntArray_set(KRef thiz, KInt index, KInt value); -KInt Kotlin_IntArray_getArrayLength(KConstRef thiz); - -KLong Kotlin_LongArray_get(KConstRef thiz, KInt index); -void Kotlin_LongArray_set(KRef thiz, KInt index, KLong value); - -KNativePtr Kotlin_NativePtrArray_get(KConstRef thiz, KInt index); -void Kotlin_NativePtrArray_set(KRef thiz, KInt index, KNativePtr value); -KInt Kotlin_NativePtrArray_getArrayLength(KConstRef thiz); - -// io/Console.kt -void Kotlin_io_Console_print(KString message); -void Kotlin_io_Console_println(KString message); void Kotlin_io_Console_println0(); -OBJ_GETTER0(Kotlin_io_Console_readLine); - -// Primitives.kt. -OBJ_GETTER(Kotlin_Int_toString, KInt value); - -// String.kt -KInt Kotlin_String_hashCode(KString thiz); -KBoolean Kotlin_String_equals(KString thiz, KConstRef other); -KInt Kotlin_String_compareTo(KString thiz, KString other); -KInt Kotlin_String_compareToIgnoreCase(KString thiz, KConstRef other); -KChar Kotlin_String_get(KString thiz, KInt index); -OBJ_GETTER(Kotlin_String_fromUtf8Array, KConstRef array, KInt start, KInt size); -OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef array, KInt start, KInt size); -OBJ_GETTER(Kotlin_String_plusImpl, KString thiz, KString other); -KInt Kotlin_String_getStringLength(KString thiz); -OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endIndex); - -OBJ_GETTER0(Kotlin_getCurrentStackTrace); - -OBJ_GETTER(Kotlin_getStackTraceStrings, KConstRef stackTrace); - -OBJ_GETTER0(Kotlin_native_internal_undefined); - -void Kotlin_native_internal_GC_suspend(KRef); -void Kotlin_native_internal_GC_resume(KRef); +void Kotlin_NativePtrArray_set(KRef thiz, KInt index, KNativePtr value); +KNativePtr Kotlin_NativePtrArray_get(KConstRef thiz, KInt index); #ifdef __cplusplus } diff --git a/runtime/src/main/cpp/ObjCExport.mm b/runtime/src/main/cpp/ObjCExport.mm index 40835a726ca..368b12ed580 100644 --- a/runtime/src/main/cpp/ObjCExport.mm +++ b/runtime/src/main/cpp/ObjCExport.mm @@ -104,6 +104,7 @@ extern "C" id Kotlin_ObjCExport_GetAssociatedObject(ObjHeader* obj) { } inline static OBJ_GETTER(AllocInstanceWithAssociatedObject, const TypeInfo* typeInfo, id associatedObject) { + // TODO: memory model! ObjHeader* result = AllocInstance(typeInfo, OBJ_RESULT); SetAssociatedObject(result, associatedObject); return result; diff --git a/runtime/src/main/cpp/Regex.cpp b/runtime/src/main/cpp/Regex.cpp index 0e2122c13b4..94a7858f2d9 100644 --- a/runtime/src/main/cpp/Regex.cpp +++ b/runtime/src/main/cpp/Regex.cpp @@ -548,6 +548,14 @@ KInt getCanonicalClass(KInt ch) { return canonicalClassesValues[index]; } +const Decomposition* getDecomposition(KInt codePoint) { + int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), codePoint); + if (decompositionKeys[index] != codePoint) { + return nullptr; + } + return &decompositionValues[index]; +} + } // namespace extern "C" { @@ -561,14 +569,6 @@ KBoolean Kotlin_text_regex_hasSingleCodepointDecompositionInternal(KInt ch) { return singleDecompositions[index] == ch; } -const Decomposition* getDecomposition(KInt codePoint) { - int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), codePoint); - if (decompositionKeys[index] != codePoint) { - return nullptr; - } - return &decompositionValues[index]; -} - OBJ_GETTER(Kotlin_text_regex_getDecompositionInternal, KInt ch) { const Decomposition* decomposition = getDecomposition(ch); if (decomposition == nullptr) { diff --git a/runtime/src/main/cpp/ToString.cpp b/runtime/src/main/cpp/ToString.cpp index ef2d2e9ee48..8123ccb6848 100644 --- a/runtime/src/main/cpp/ToString.cpp +++ b/runtime/src/main/cpp/ToString.cpp @@ -76,8 +76,7 @@ OBJ_GETTER(Kotlin_Byte_toString, KByte value) { } OBJ_GETTER(Kotlin_Char_toString, KChar value) { - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, 1, OBJ_RESULT)->array(); + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, 1, OBJ_RESULT)->array(); *CharArrayAddressOfElementAt(result, 0) = value; RETURN_OBJ(result->obj()); } diff --git a/runtime/src/main/kotlin/kotlin/String.kt b/runtime/src/main/kotlin/kotlin/String.kt index 1f55e65febe..5e31807da15 100644 --- a/runtime/src/main/kotlin/kotlin/String.kt +++ b/runtime/src/main/kotlin/kotlin/String.kt @@ -46,7 +46,7 @@ public final class String : Comparable, CharSequence { external private fun getStringLength(): Int @SymbolName("Kotlin_String_plusImpl") - external private fun plusImpl(other: Any): String + external private fun plusImpl(other: String): String @SymbolName("Kotlin_String_equals") external public override fun equals(other: Any?): Boolean diff --git a/runtime/src/main/kotlin/kotlin/text/Strings.kt b/runtime/src/main/kotlin/kotlin/text/Strings.kt index 4f76810a761..9db8a26fcbd 100644 --- a/runtime/src/main/kotlin/kotlin/text/Strings.kt +++ b/runtime/src/main/kotlin/kotlin/text/Strings.kt @@ -55,7 +55,6 @@ internal external fun stringEqualsIgnoreCase(thiz: String, other: String): Boole public actual external fun String.replace( oldChar: Char, newChar: Char, ignoreCase: Boolean): String - /** * Returns a new string obtained by replacing all occurrences of the [oldValue] substring in this string * with the specified [newValue] string. diff --git a/runtime/src/relaxed/cpp/MemoryImpl.cpp b/runtime/src/relaxed/cpp/MemoryImpl.cpp new file mode 100644 index 00000000000..ca8f8f2609c --- /dev/null +++ b/runtime/src/relaxed/cpp/MemoryImpl.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +#include "Memory.h" + +// Note that only C++ part of the runtime goes via those functions, Kotlin uses specialized versions. + +extern "C" { + +OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) { + RETURN_RESULT_OF(AllocInstanceRelaxed, typeInfo); +} + +OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elements) { + RETURN_RESULT_OF(AllocArrayInstanceRelaxed, typeInfo, elements); +} + +OBJ_GETTER(InitInstance, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitInstanceRelaxed, location, typeInfo, ctor); +} + +OBJ_GETTER(InitSharedInstance, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitSharedInstanceRelaxed, location, localLocation, typeInfo, ctor); +} + +} // extern "C" diff --git a/runtime/src/strict/cpp/MemoryImpl.cpp b/runtime/src/strict/cpp/MemoryImpl.cpp new file mode 100644 index 00000000000..06672150920 --- /dev/null +++ b/runtime/src/strict/cpp/MemoryImpl.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +#include "Memory.h" + +// Note that only C++ part of the runtime goes via those functions, Kotlin uses specialized versions. + +extern "C" { + +OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) { + RETURN_RESULT_OF(AllocInstanceStrict, typeInfo); +} + +OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elements) { + RETURN_RESULT_OF(AllocArrayInstanceStrict, typeInfo, elements); +} + +OBJ_GETTER(InitInstance, + ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitInstanceStrict, location, typeInfo, ctor); +} + +OBJ_GETTER(InitSharedInstance, + ObjHeader** location, ObjHeader** localLocation, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + RETURN_RESULT_OF(InitSharedInstanceStrict, location, localLocation, typeInfo, ctor); +} + +} // extern "C"