From d072920133c087037ded0ee61a6c19a7372787a4 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 9 Apr 2019 15:40:14 +0200 Subject: [PATCH] Rework memory manager to make stack slot update faster (#2813) --- .../JsRuntime/src/main/kotlin/jsinterop.kt | 24 +- .../kotlin/backend/konan/CAdapterGenerator.kt | 22 +- .../backend/konan/llvm/CodeGenerator.kt | 2 +- .../kotlin/backend/konan/llvm/IrToBitcode.kt | 5 +- .../backend/konan/llvm/RetainAnnotation.kt | 19 +- backend.native/tests/build.gradle | 3 +- backend.native/tests/interop/objc/smoke.kt | 2 + .../tests/produce_dynamic/simple/hello.kt | 4 + .../tests/produce_dynamic/simple/main.c | 5 + backend.native/tests/runtime/memory/weak0.kt | 5 +- runtime/src/launcher/cpp/launcher.cpp | 5 +- runtime/src/main/cpp/Atomic.cpp | 2 +- runtime/src/main/cpp/Common.h | 1 + runtime/src/main/cpp/Exceptions.cpp | 16 +- runtime/src/main/cpp/KString.cpp | 13 +- runtime/src/main/cpp/Memory.cpp | 976 ++++++++++++------ runtime/src/main/cpp/Memory.h | 208 ++-- runtime/src/main/cpp/Worker.cpp | 80 +- .../main/kotlin/kotlin/native/Annotations.kt | 8 + 19 files changed, 882 insertions(+), 518 deletions(-) diff --git a/Interop/JsRuntime/src/main/kotlin/jsinterop.kt b/Interop/JsRuntime/src/main/kotlin/jsinterop.kt index 7eee3eb0967..24b264ae863 100644 --- a/Interop/JsRuntime/src/main/kotlin/jsinterop.kt +++ b/Interop/JsRuntime/src/main/kotlin/jsinterop.kt @@ -27,15 +27,15 @@ typealias Pointer = Int /** * @Retain annotation is required to preserve functions from internalization and DCE. */ -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_allocateArena") external public fun allocateArena(): Arena -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_freeArena") external public fun freeArena(arena: Arena) -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_pushIntToArena") external public fun pushIntToArena(arena: Arena, value: Int) @@ -49,15 +49,15 @@ fun doubleUpper(value: Double): Int = fun doubleLower(value: Double): Int = (value.toBits() and 0x00000000ffffffff) .toInt() -@Retain +@RetainForTarget("wasm") @SymbolName("ReturnSlot_getDouble") external public fun ReturnSlot_getDouble(): Double -@Retain +@RetainForTarget("wasm") @SymbolName("Kotlin_String_utf16pointer") external public fun stringPointer(message: String): Pointer -@Retain +@RetainForTarget("wasm") @SymbolName("Kotlin_String_utf16length") external public fun stringLengthBytes(message: String): Int @@ -68,7 +68,7 @@ fun wrapFunction(func: KtFunction): Int { return ptr.toInt() // TODO: LP64 unsafe. } -@Retain +@RetainForTarget("wasm") @ExportForCppRuntime("Konan_js_runLambda") fun runLambda(pointer: Int, argumentsArena: Arena, argumentsArenaSize: Int): Int { val arguments = arrayListOf() @@ -104,19 +104,19 @@ open class JsArray(arena: Arena, index: Object): JsValue(arena, index) { get() = this.getInt("length") } -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_getInt") external public fun getInt(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int; -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_getProperty") external public fun Konan_js_getProperty(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int; -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_setFunction") external public fun setFunction(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int , function: Int) -@Retain +@RetainForTarget("wasm") @SymbolName("Konan_js_setString") external public fun setString(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int, stringPtr: Pointer, stringLength: Int ) @@ -140,4 +140,4 @@ fun JsValue.setter(property: String, string: String) { object ArenaManager { val globalArena = allocateArena() var currentArena = globalArena -} +} \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt index 2ff025811c3..83b19159144 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt @@ -850,11 +850,14 @@ internal class CAdapterGenerator( | |extern "C" { |void UpdateHeapRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW; + |void UpdateStackRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW; |KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW; |KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW; |void* CreateStablePointer(KObjHeader*) RUNTIME_NOTHROW; |void DisposeStablePointer(void*) RUNTIME_NOTHROW; |int IsInstance(const KObjHeader*, const KTypeInfo*) RUNTIME_NOTHROW; + |void EnterFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; + |void LeaveFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; |void Kotlin_initRuntimeIfNeeded(); | |KObjHeader* CreateStringFromCString(const char*, KObjHeader**); @@ -862,19 +865,32 @@ internal class CAdapterGenerator( |void DisposeCString(char* cstring); |} // extern "C" | + |struct ${prefix}_FrameOverlay { + | void* arena; + | ${prefix}_FrameOverlay* previous; + | ${prefix}_KInt parameters; + | ${prefix}_KInt count; + |}; + | |class KObjHolder { |public: - | KObjHolder() : obj_(nullptr) {} + | KObjHolder() : obj_(nullptr) { + | EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*)); + | } | explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) { - | UpdateHeapRef(&obj_, obj); + | EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*)); + | UpdateStackRef(&obj_, obj); | } | ~KObjHolder() { - | UpdateHeapRef(&obj_, nullptr); + | LeaveFrame(frame(), 0, sizeof(*this)/sizeof(void*)); | } | KObjHeader* obj() { return obj_; } | KObjHeader** slot() { return &obj_; } | private: + | ${prefix}_FrameOverlay frame_; | KObjHeader* obj_; + | + | KObjHeader** frame() { return reinterpret_cast(&frame_); } |}; |static void DisposeStablePointerImpl(${prefix}_KNativePtr ptr) { | DisposeStablePointer(ptr); diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 31f88d96bd8..18709e97ce5 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -996,7 +996,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, private val kotlinExceptionRtti: ConstPointer get() = constPointer(importGlobal( - "_ZTI9ObjHolder", // typeinfo for ObjHolder + "_ZTI18ExceptionObjHolder", // typeinfo for ObjHolder int8TypePtr, origin = context.stdlibModule.llvmSymbolOrigin )).bitcast(int8TypePtr) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index e89f81e0e57..5ba675ba9e6 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -13,6 +13,7 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.ir.* import org.jetbrains.kotlin.backend.konan.llvm.coverage.* +import org.jetbrains.kotlin.backend.konan.llvm.objcexport.is64Bit import org.jetbrains.kotlin.backend.konan.optimizations.* import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.UnsignedType @@ -681,7 +682,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map NSObject) { createAndTestWeakReference(block) } + kotlin.native.internal.GC.collect() + assertNull(ref.get()) } diff --git a/backend.native/tests/produce_dynamic/simple/hello.kt b/backend.native/tests/produce_dynamic/simple/hello.kt index 27347ed375b..f21a508c398 100644 --- a/backend.native/tests/produce_dynamic/simple/hello.kt +++ b/backend.native/tests/produce_dynamic/simple/hello.kt @@ -14,6 +14,10 @@ fun hello() { fun getString() = "Kotlin/Native" +data class Data(var string: String) + +fun getMutable() = Data("foo") + // Class with inheritance. open class Base { open fun foo() = println("Base.foo") diff --git a/backend.native/tests/produce_dynamic/simple/main.c b/backend.native/tests/produce_dynamic/simple/main.c index 76b3ef980f7..c89c6293417 100644 --- a/backend.native/tests/produce_dynamic/simple/main.c +++ b/backend.native/tests/produce_dynamic/simple/main.c @@ -16,9 +16,11 @@ int main(void) { T_(I) casted_impl2 = { .pinned = impl2.pinned }; T_(Enum) enum1 = __ kotlin.root.Enum.HUNDRED.get(); T_(Codeable) object1 = __ kotlin.root.get_an_object(); + T_(Data) data = __ kotlin.root.getMutable(); const char* string1 = __ kotlin.root.getString(); const char* string2 = __ kotlin.root.Singleton.toString(singleton); + const char* string3 = __ kotlin.root.Data.get_string(data); __ kotlin.root.hello(); __ kotlin.root.Base.foo(base); @@ -40,6 +42,8 @@ int main(void) { printf("singleton = %s\n", string2); + printf("mutable = %s\n", string3); + topLevelFunctionVoidFromC(42, 0); __ kotlin.root.topLevelFunctionVoid(42, 0); printf("topLevel = %d %d\n", topLevelFunctionFromC(780, 3), __ kotlin.root.topLevelFunctionFromCShort(5, 2)); @@ -49,6 +53,7 @@ int main(void) { __ DisposeStablePointer(singleton.pinned); __ DisposeString(string1); __ DisposeString(string2); + __ DisposeString(string3); __ DisposeStablePointer(base.pinned); __ DisposeStablePointer(child.pinned); __ DisposeStablePointer(impl1.pinned); diff --git a/backend.native/tests/runtime/memory/weak0.kt b/backend.native/tests/runtime/memory/weak0.kt index 71deab0167e..b0eac53d4e3 100644 --- a/backend.native/tests/runtime/memory/weak0.kt +++ b/backend.native/tests/runtime/memory/weak0.kt @@ -13,7 +13,6 @@ data class Data(val s: String) fun localWeak(): WeakReference { val x = Data("Hello") val weak = WeakReference(x) - println(weak.get()) return weak } @@ -29,10 +28,14 @@ fun multiWeak(): Array> { @Test fun runTest() { val weak = localWeak() + kotlin.native.internal.GC.collect() val value = weak.get() println(value?.toString()) val weaks = multiWeak() + + kotlin.native.internal.GC.collect() + weaks.forEach { it -> if (it.get()?.s != null) throw Error("not null") } diff --git a/runtime/src/launcher/cpp/launcher.cpp b/runtime/src/launcher/cpp/launcher.cpp index 8ce022c375b..560b557f3ee 100644 --- a/runtime/src/launcher/cpp/launcher.cpp +++ b/runtime/src/launcher/cpp/launcher.cpp @@ -29,8 +29,9 @@ OBJ_GETTER(setupArgs, int argc, const char** argv) { ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT); ArrayHeader* array = result->array(); for (int index = 1; index < argc; index++) { - CreateStringFromCString( - argv[index], ArrayAddressOfElementAt(array, index - 1)); + ObjHolder result; + CreateStringFromCString(argv[index], result.slot()); + UpdateHeapRef(ArrayAddressOfElementAt(array, index - 1), result.obj()); } return result; } diff --git a/runtime/src/main/cpp/Atomic.cpp b/runtime/src/main/cpp/Atomic.cpp index 1ee6cd5b6d9..92561ee0cbc 100644 --- a/runtime/src/main/cpp/Atomic.cpp +++ b/runtime/src/main/cpp/Atomic.cpp @@ -208,7 +208,7 @@ OBJ_GETTER(Kotlin_AtomicReference_get, KRef thiz) { // rescheduled unluckily, between the moment value is read from the field and RC is incremented, // object may go away. AtomicReferenceLayout* ref = asAtomicReference(thiz); - RETURN_RESULT_OF(ReadRefLocked, &ref->value_, &ref->lock_); + RETURN_RESULT_OF(ReadHeapRefLocked, &ref->value_, &ref->lock_); } } // extern "C" diff --git a/runtime/src/main/cpp/Common.h b/runtime/src/main/cpp/Common.h index 15d954f2aea..8be6483ac05 100644 --- a/runtime/src/main/cpp/Common.h +++ b/runtime/src/main/cpp/Common.h @@ -24,6 +24,7 @@ #define RUNTIME_USED __attribute__((used)) #define ALWAYS_INLINE __attribute__((always_inline)) +#define NO_INLINE __attribute__((noinline)) #if KONAN_NO_THREADS #define THREAD_LOCAL_VARIABLE diff --git a/runtime/src/main/cpp/Exceptions.cpp b/runtime/src/main/cpp/Exceptions.cpp index c903d5809ab..520b952d254 100644 --- a/runtime/src/main/cpp/Exceptions.cpp +++ b/runtime/src/main/cpp/Exceptions.cpp @@ -156,7 +156,9 @@ OBJ_GETTER0(GetCurrentStackTrace) { OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) { #if OMIT_BACKTRACE ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, 1, OBJ_RESULT); - CreateStringFromCString("", ArrayAddressOfElementAt(result->array(), 0)); + ObjHolder holder; + CreateStringFromCString("", holder.slot()); + UpdateHeapRef(ArrayAddressOfElementAt(result->array(), 0), holder.obj()); return result; #else uint32_t size = stackTrace->array()->count_; @@ -172,7 +174,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) { } char line[512]; konan::snprintf(line, sizeof(line) - 1, "%s (%p)", symbol, (void*)(intptr_t)address); - CreateStringFromCString(line, ArrayAddressOfElementAt(strings->array(), index)); + ObjHolder holder; + CreateStringFromCString(line, holder.slot()); + UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj()); } #else if (size > 0) { @@ -195,7 +199,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) { } else { result = symbol; } - CreateStringFromCString(result, ArrayAddressOfElementAt(strings->array(), index)); + ObjHolder holder; + CreateStringFromCString(result, holder.slot()); + UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj()); } } #endif @@ -210,7 +216,7 @@ void ThrowException(KRef exception) { PrintThrowable(exception); RuntimeCheck(false, "Exceptions unsupported"); #else - throw ObjHolder(exception); + throw ExceptionObjHolder(exception); #endif } @@ -276,7 +282,7 @@ static void KonanTerminateHandler() { } else { try { std::rethrow_exception(currentException); - } catch (ObjHolder& e) { + } catch (ExceptionObjHolder& e) { TerminateWithUnhandledException(e.obj()); } catch (...) { // Not a Kotlin exception. diff --git a/runtime/src/main/cpp/KString.cpp b/runtime/src/main/cpp/KString.cpp index 20239307978..57bb4f730b2 100644 --- a/runtime/src/main/cpp/KString.cpp +++ b/runtime/src/main/cpp/KString.cpp @@ -1176,8 +1176,7 @@ OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endInd RETURN_RESULT_OF0(TheEmptyString); } KInt length = endIndex - startIndex; - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, length, OBJ_RESULT)->array(); + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, length, OBJ_RESULT)->array(); memcpy(CharArrayAddressOfElementAt(result, 0), CharArrayAddressOfElementAt(thiz, startIndex), length * sizeof(KChar)); @@ -1185,14 +1184,14 @@ OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endInd } const KChar* Kotlin_String_utf16pointer(KString message) { - RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string"); - const KChar* utf16 = CharArrayAddressOfElementAt(message, 0); - return utf16; + RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string"); + const KChar* utf16 = CharArrayAddressOfElementAt(message, 0); + return utf16; } KInt Kotlin_String_utf16length(KString message) { - RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string"); - return message->count_ * sizeof(KChar); + RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string"); + return message->count_ * sizeof(KChar); } diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index ff3595fbb3a..9e5039ba072 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -36,6 +36,8 @@ #define USE_GC 1 // Define to 1 to print all memory operations. #define TRACE_MEMORY 0 +// Define to 1 to print major GC events. +#define TRACE_GC 0 // Collect memory manager events statistics. #define COLLECT_STATISTIC 0 // Auto-adjust GC thresholds. @@ -52,15 +54,23 @@ constexpr container_size_t kObjectAlignment = 8; static_assert(sizeof(ContainerHeader) % kObjectAlignment == 0, "sizeof(ContainerHeader) is not aligned"); #if TRACE_MEMORY +#undef TRACE_GC +#define TRACE_GC 1 #define MEMORY_LOG(...) konan::consolePrintf(__VA_ARGS__); #else #define MEMORY_LOG(...) #endif +#if TRACE_GC +#define GC_LOG(...) konan::consolePrintf(__VA_ARGS__); +#else +#define GC_LOG(...) +#endif + #if USE_GC // Collection threshold default (collect after having so many elements in the // release candidates set). -constexpr size_t kGcThreshold = 4 * 1024; +constexpr size_t kGcThreshold = 16 * 1024; #if GC_ERGONOMICS // Ergonomic thresholds. // If GC to computations time ratio is above that value, @@ -69,21 +79,18 @@ constexpr double kGcToComputeRatioThreshold = 0.5; // Never exceed this value when increasing GC threshold. constexpr size_t kMaxErgonomicThreshold = 1024 * 1024; #endif // GC_ERGONOMICS - -typedef KStdDeque ContainerHeaderDeque; -#endif +// Threshold of size for toFree set, triggering actual cycle collector. +constexpr size_t kMaxToFreeSize = 8 * 1024; +// How many elements in finalizer queue allowed before cleaning it up. +constexpr size_t kFinalizerQueueThreshold = 32; +#endif // USE_GC } // namespace -#if TRACE_MEMORY || USE_GC typedef KStdUnorderedSet ContainerHeaderSet; typedef KStdVector ContainerHeaderList; typedef KStdVector KRefPtrList; -#endif - -struct FrameOverlay { - ArenaContainer* arena; -}; +typedef KStdDeque ContainerHeaderDeque; // A little hack that allows to enable -O2 optimizations // Prevents clang from replacing FrameOverlay struct @@ -96,7 +103,10 @@ volatile int allocCount = 0; volatile int aliveMemoryStatesCount = 0; // Forward declarations. -void FreeContainer(ContainerHeader* header); +void freeContainer(ContainerHeader* header) NO_INLINE; +#if USE_GC +void garbageCollect(MemoryState* state, bool force) NO_INLINE; +#endif // USE_GC #if COLLECT_STATISTIC class MemoryStatistic { @@ -148,7 +158,6 @@ public: void incReleaseRef(const ContainerHeader* header, bool atomic, bool cyclic, int stack) { if (atomic) { - RuntimeAssert(!cyclic, "Atomic updates cannot be cyclic yet"); atomicReleaseRefs++; } else { if (cyclic) releaseCyclicRefs++; else releaseRefs++; @@ -295,10 +304,12 @@ struct MemoryState { ContainerHeaderList* roots; // Real candidates excluding those with refcount = 0. // How many GC suspend requests happened. int gcSuspendCount; - // How many candidate elements in toFree shall trigger collection. + // How many candidate elements in toRelease shall trigger collection. size_t gcThreshold; // If collection is in progress. bool gcInProgress; + // Objects to be released. + ContainerHeaderList* toRelease; #if GC_ERGONOMICS uint64_t lastGcTimestamp; @@ -356,7 +367,7 @@ struct MemoryState { #define OBJECT_ALLOC_TRACE(state, size, object) \ MEMORY_LOG("Object alloc %d at %p\n", size, object) #define UPDATE_REF_TRACE(state, oldRef, newRef, slot, stack) \ - MEMORY_LOG("UpdateRef %s*%p: %p -> %p\n", stack ? "stack " : "", slot, oldRef, newRef) + MEMORY_LOG("UpdateRef %s*%p: %p -> %p\n", stack ? "stack " : "heap ", slot, oldRef, newRef) // Events macro definitions. // Called on worker's memory init. @@ -392,8 +403,94 @@ struct MemoryState { namespace { +// Container for a single object. +class ObjectContainer : public Container { + public: + // Single instance. + explicit ObjectContainer(MemoryState* state, const TypeInfo* type_info) { + Init(state, type_info); + } + + // Object container shalln't have any dtor, as it's being freed by + // ::Release(). + + ObjHeader* GetPlace() const { + return reinterpret_cast(header_ + 1); + } + + private: + void Init(MemoryState* state, const TypeInfo* type_info); +}; + + +class ArrayContainer : public Container { + public: + ArrayContainer(MemoryState* state, const TypeInfo* type_info, uint32_t elements) { + Init(state, type_info, elements); + } + + // Array container shalln't have any dtor, as it's being freed by ::Release(). + + ArrayHeader* GetPlace() const { + return reinterpret_cast(header_ + 1); + } + + private: + void Init(MemoryState* state, const TypeInfo* type_info, uint32_t elements); +}; + +// Class representing arena-style placement container. +// Container is used for reference counting, and it is assumed that objects +// with related placement will share container. Only +// whole container can be freed, individual objects are not taken into account. +class ArenaContainer; + +struct ContainerChunk { + ContainerChunk* next; + ArenaContainer* arena; + // Then we have ContainerHeader here. + ContainerHeader* asHeader() { + return reinterpret_cast(this + 1); + } +}; + +class ArenaContainer { + public: + void Init(); + void Deinit(); + + // Place individual object in this container. + ObjHeader* PlaceObject(const TypeInfo* type_info); + + // Places an array of certain type in this container. Note that array_type_info + // is type info for an array, not for an individual element. Also note that exactly + // same operation could be used to place strings. + ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count); + + ObjHeader** getSlot(); + + private: + void* place(container_size_t size); + + bool allocContainer(container_size_t minSize); + + void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) { + obj->typeInfoOrMeta_ = const_cast(typeInfo); + obj->setContainer(currentChunk_->asHeader()); + // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC. + } + + ContainerChunk* currentChunk_; + uint8_t* current_; + uint8_t* end_; + ArrayHeader* slots_; + uint32_t slotsCount_; +}; + + // TODO: can we pass this variable as an explicit argument? THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr; +THREAD_LOCAL_VARIABLE FrameOverlay* currentFrame = nullptr; constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**); @@ -413,6 +510,11 @@ inline container_size_t alignUp(container_size_t size, int alignment) { return (size + alignment - 1) & ~(alignment - 1); } +inline ContainerHeader* realFrozenContainer(ContainerHeader* container) { + RuntimeAssert(container->frozen(), "Only makes sense on frozen objects"); + return reinterpret_cast(container + 1)->container(); +} + inline uint32_t arrayObjectSize(const TypeInfo* typeInfo, uint32_t count) { // Note: array body is aligned, but for size computation it is enough to align the sum. static_assert(kObjectAlignment % alignof(KLong) == 0, ""); @@ -435,15 +537,6 @@ inline container_size_t objectSize(const ObjHeader* obj) { return alignUp(size, kObjectAlignment); } -inline bool isArenaSlot(ObjHeader** slot) { - return (reinterpret_cast(slot) & ARENA_BIT) != 0; -} - -inline ObjHeader** asArenaSlot(ObjHeader** slot) { - return reinterpret_cast( - reinterpret_cast(slot) & ~ARENA_BIT); -} - inline FrameOverlay* asFrameOverlay(ObjHeader** slot) { return reinterpret_cast(slot); } @@ -490,7 +583,7 @@ RUNTIME_NORETURN void ThrowFreezingException(KRef toFreeze, KRef blocker); } // extern "C" -inline void runDeallocationHooks(ContainerHeader* container) { +void runDeallocationHooks(ContainerHeader* container) { ObjHeader* obj = reinterpret_cast(container + 1); for (int index = 0; index < container->objectCount(); index++) { @@ -544,8 +637,6 @@ inline void traverseContainerReferredObjects(ContainerHeader* container, func pr }); } -#if USE_GC - inline bool isMarkedAsRemoved(ContainerHeader* container) { return (reinterpret_cast(container) & 1) != 0; } @@ -559,7 +650,9 @@ inline ContainerHeader* clearRemoved(ContainerHeader* container) { reinterpret_cast(container) & ~static_cast(1)); } -inline void processFinalizerQueue(MemoryState* state) { +#if USE_GC + +void processFinalizerQueue(MemoryState* state) { // TODO: reuse elements of finalizer queue for new allocations. while (state->finalizerQueue != nullptr) { auto* container = state->finalizerQueue; @@ -576,14 +669,15 @@ inline void processFinalizerQueue(MemoryState* state) { } #endif -inline void scheduleDestroyContainer(MemoryState* state, ContainerHeader* container) { +void scheduleDestroyContainer(MemoryState* state, ContainerHeader* container) { #if USE_GC RuntimeAssert(container != nullptr, "Cannot destroy null container"); container->setNextLink(state->finalizerQueue); state->finalizerQueue = container; state->finalizerQueueSize++; // We cannot clean finalizer queue while in GC. - if (!state->gcInProgress && state->finalizerQueueSuspendCount == 0 && state->finalizerQueueSize > 256) { + if (!state->gcInProgress && state->finalizerQueueSuspendCount == 0 && + state->finalizerQueueSize >= kFinalizerQueueThreshold) { processFinalizerQueue(state); } #else @@ -603,30 +697,39 @@ inline void IncrementRC(ContainerHeader* container) { template inline void DecrementRC(ContainerHeader* container) { if (container->decRefCount() == 0) { - FreeContainer(container); + freeContainer(container); } } +inline void DecrementRC(ContainerHeader* container) { + if (Shareable(container)) + DecrementRC(container); + else + DecrementRC(container); +} + +template +inline void EnqueueDecrementRC(ContainerHeader* container) { + RuntimeCheck(false, "Not yet implemeneted"); +} + #else // USE_GC -inline uint32_t freeableSize(MemoryState* state) { - return state->toFree->size(); -} - template inline void IncrementRC(ContainerHeader* container) { container->incRefCount(); - container->setColorUnlessGreen(CONTAINER_TAG_GC_BLACK); } template inline void DecrementRC(ContainerHeader* container) { + // TODO: enable me, once account for inner references in frozen objects correctly. + // RuntimeAssert(container->refCount() > 0, "Must be positive"); if (container->decRefCount() == 0) { - FreeContainer(container); + freeContainer(container); } else if (UseCycleCollector) { // Possible root. - RuntimeAssert(!Atomic, "Cycle collector shalln't be used with shared objects yet"); - RuntimeAssert(container->objectCount() == 1, - "cycle collector shall only work with single object containers"); + RuntimeAssert(container->refCount() > 0, "Must be positive"); + RuntimeAssert(!Atomic && !container->shareable(), "Cycle collector shalln't be used with shared objects yet"); + RuntimeAssert(container->objectCount() == 1, "cycle collector shall only work with single object containers"); // We do not use cycle collector for frozen objects, as we already detected // possible cycles during freezing. // Also do not use cycle collector for provable acyclic objects. @@ -634,21 +737,73 @@ inline void DecrementRC(ContainerHeader* container) { if (color != CONTAINER_TAG_GC_PURPLE && color != CONTAINER_TAG_GC_GREEN) { container->setColorAssertIfGreen(CONTAINER_TAG_GC_PURPLE); if (!container->buffered()) { + auto* state = memoryState; container->setBuffered(); - auto state = memoryState; - state->toFree->push_back(container); - if (state->gcSuspendCount == 0 && freeableSize(state) >= state->gcThreshold) { - GarbageCollect(); + if (state->toFree != nullptr) { + state->toFree->push_back(container); + MEMORY_LOG("toFree is now %d\n", state->toFree->size()) + if (state->gcSuspendCount == 0 && state->toRelease->size() >= state->gcThreshold) { + GC_LOG("Calling GC from DecrementRC: %d\n", state->toRelease->size()) + garbageCollect(state, false); + } } } } } } -inline void initThreshold(MemoryState* state, uint32_t gcThreshold) { - state->gcThreshold = gcThreshold; - state->toFree->reserve(gcThreshold); +inline void DecrementRC(ContainerHeader* container) { + auto* state = memoryState; + RuntimeAssert(state->gcInProgress, "Must only be called during GC"); + // TODO: enable me, once account for inner references in frozen objects correctly. + // RuntimeAssert(container->refCount() > 0, "Must be positive"); + bool useCycleCollector = container->tag() == CONTAINER_TAG_NORMAL; + if (container->decRefCount() == 0) { + freeContainer(container); + } else if (useCycleCollector && state->toFree != nullptr) { + RuntimeAssert(container->refCount() > 0, "Must be positive"); + RuntimeAssert(!container->shareable(), "Cycle collector shalln't be used with shared objects yet"); + RuntimeAssert(container->objectCount() == 1, "cycle collector shall only work with single object containers"); + // We do not use cycle collector for frozen objects, as we already detected + // possible cycles during freezing. + // Also do not use cycle collector for provable acyclic objects. + int color = container->color(); + if (color != CONTAINER_TAG_GC_PURPLE && color != CONTAINER_TAG_GC_GREEN) { + container->setColorAssertIfGreen(CONTAINER_TAG_GC_PURPLE); + if (!container->buffered()) { + container->setBuffered(); + state->toFree->push_back(container); + } + } + } } + +template +inline void EnqueueDecrementRC(ContainerHeader* container) { + auto* state = memoryState; + if (CanCollect) { + if (state->toRelease->size() >= state->gcThreshold && state->gcSuspendCount == 0) { + GC_LOG("Calling GC from EnqueueDecrementRC: %d\n", state->toRelease->size()) + garbageCollect(state, false); + } + } + state->toRelease->push_back(container); +} + +inline void initGcThreshold(MemoryState* state, uint32_t gcThreshold) { + state->gcThreshold = gcThreshold; + state->toRelease->reserve(gcThreshold); +} + +#if GC_ERGONOMICS +inline void increaseGcThreshold(MemoryState* state) { + auto newThreshold = state->gcThreshold * 3 / 2 + 1; + if (newThreshold <= kMaxErgonomicThreshold) { + initGcThreshold(state, newThreshold); + } +} +#endif // GC_ERGONOMICS + #endif // USE_GC #if TRACE_MEMORY && USE_GC @@ -669,14 +824,18 @@ void dumpObject(ObjHeader* ref, int indent) { } void dumpContainerContent(ContainerHeader* container) { + if (container->refCount() <= 0) { + MEMORY_LOG("%p has non-positive RC, likely a memory bug\n", container) + return; + } if (isAggregatingFrozenContainer(container)) { MEMORY_LOG("%s aggregating container %p with %d objects rc=%d\n", colorNames[container->color()], container, container->objectCount(), container->refCount()); ContainerHeader** subContainer = reinterpret_cast(container + 1); for (int i = 0; i < container->objectCount(); ++i) { - ObjHeader* obj = reinterpret_cast(subContainer + 1); - MEMORY_LOG(" object %p of type %p: ", obj, obj->type_info()); - dumpObject(obj, 4); + ContainerHeader* sub = *subContainer++; + MEMORY_LOG(" container %p\n ", sub); + dumpContainerContent(sub); } } else { MEMORY_LOG("%s regular %s%scontainer %p with %d objects rc=%d\n", @@ -693,13 +852,15 @@ void dumpContainerContent(ContainerHeader* container) { void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) { dumpContainerContent(header); seen->insert(header); - traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) { - auto* child = ref->container(); - RuntimeAssert(!isArena(child), "A reference to local object is encountered"); - if (child != nullptr && (seen->count(child) == 0)) { - dumpWorker(prefix, child, seen); - } - }); + if (!isAggregatingFrozenContainer(header)) { + traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) { + auto* child = ref->container(); + RuntimeAssert(!isArena(child), "A reference to local object is encountered"); + if (child != nullptr && (seen->count(child) == 0)) { + dumpWorker(prefix, child, seen); + } + }); + } } void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { @@ -893,10 +1054,10 @@ inline bool canBeCyclic(ContainerHeader* container) { return true; } -ALWAYS_INLINE inline void addRef(ContainerHeader* container) { - // Looking at container type we may want to skip AddRef() totally - // (non-escaping stack objects, constant objects). - switch (container->refCount_ & CONTAINER_TAG_MASK) { +inline void AddHeapRef(ContainerHeader* container) { + MEMORY_LOG("AddHeapRef %p: rc=%d\n", container, container->refCount()) + UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 0) + switch (container->tag()) { case CONTAINER_TAG_STACK: break; case CONTAINER_TAG_NORMAL: @@ -909,60 +1070,59 @@ ALWAYS_INLINE inline void addRef(ContainerHeader* container) { } } -ALWAYS_INLINE inline void AddHeapRef(ContainerHeader* container) { - UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 0); - addRef(container); -} - -ALWAYS_INLINE inline void AddHeapRef(ObjHeader* header) { +inline void AddHeapRef(const ObjHeader* header) { auto* container = header->container(); - if (container != nullptr) AddHeapRef(container); + if (container != nullptr) + AddHeapRef(const_cast(container)); } -ALWAYS_INLINE inline void AddStackRef(ContainerHeader* container) { +inline void AddStackRef(ContainerHeader* container) { UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 1); - addRef(container); -} - -ALWAYS_INLINE inline void AddStackRef(ObjHeader* header) { - auto* container = header->container(); - if (container != nullptr) AddStackRef(container); -} - -ALWAYS_INLINE inline void releaseRef(ContainerHeader* header) { - // Looking at container type we may want to skip ReleaseRef() totally - // (non-escaping stack objects, constant objects). - switch (header->tag()) { - case CONTAINER_TAG_STACK: - break; - case CONTAINER_TAG_NORMAL: - DecrementRC(header); - break; - /* case CONTAINER_TAG_FROZEN: case CONTAINER_TAG_ATOMIC: */ - default: - DecrementRC(header); - break; + if (container->shareable()) { + IncrementRC(container); } } -ALWAYS_INLINE inline void ReleaseHeapRef(ContainerHeader* container) { - UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 0); - releaseRef(container); +inline void AddStackRef(const ObjHeader* header) { + auto* container = header->container(); + if (container != nullptr) { + AddStackRef(const_cast(container)); + } } -ALWAYS_INLINE inline void ReleaseStackRef(ContainerHeader* container) { +inline void ReleaseHeapRef(ContainerHeader* container) { + MEMORY_LOG("ReleaseHeapRef %p: rc=%d\n", container, container->refCount()) + UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 0) + switch (container->tag()) { + case CONTAINER_TAG_STACK: + break; + case CONTAINER_TAG_NORMAL: + EnqueueDecrementRC(container); + break; + /* case CONTAINER_TAG_FROZEN: case CONTAINER_TAG_ATOMIC: */ + default: + DecrementRC(container); + break; + } +} + +inline void ReleaseStackRef(ContainerHeader* container) { UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 1); - releaseRef(container); + if (container->shareable() && container->decRefCount() == 0) { + freeContainer(container); + } } -ALWAYS_INLINE inline void ReleaseHeapRef(ObjHeader* header) { +inline void ReleaseHeapRef(const ObjHeader* header) { auto* container = header->container(); - if (container != nullptr) ReleaseHeapRef(container); + if (container != nullptr) + ReleaseHeapRef(const_cast(container)); } -ALWAYS_INLINE inline void ReleaseStackRef(ObjHeader* header) { +inline void ReleaseStackRef(const ObjHeader* header) { auto* container = header->container(); - if (container != nullptr) ReleaseStackRef(container); + if (container != nullptr) + ReleaseStackRef(const_cast(container)); } // We use first slot as place to store frame-local arena container. @@ -970,7 +1130,7 @@ ALWAYS_INLINE inline void ReleaseStackRef(ObjHeader* header) { // do two allocations per frame (ArenaContainer + actual container). inline ArenaContainer* initedArena(ObjHeader** auxSlot) { auto frame = asFrameOverlay(auxSlot); - auto arena = frame->arena; + auto arena = reinterpret_cast(frame->arena); if (!arena) { arena = konanConstructInstance(); MEMORY_LOG("Initializing arena in %p\n", frame) @@ -986,8 +1146,7 @@ inline size_t containerSize(const ContainerHeader* container) { for (int object = 0; object < container->objectCount(); object++) { size_t size = objectSize(obj); result += size; - obj = reinterpret_cast( - reinterpret_cast(obj) + size); + obj = reinterpret_cast(reinterpret_cast(obj) + size); } return result; } @@ -1027,24 +1186,43 @@ void ObjHeader::destroyMetaObject(TypeInfo** location) { konanFreeMemory(meta); } -ContainerHeader* AllocContainer(size_t size) { - auto state = memoryState; +ContainerHeader* AllocContainer(MemoryState* state, size_t size) { + ContainerHeader* result = nullptr; #if USE_GC - // TODO: try to reuse elements of finalizer queue for new allocations, question - // is how to get actual size of container. + // We recycle elements of finalizer queue for new allocations, to avoid trashing memory manager. + ContainerHeader *container = state->finalizerQueue, *previous = nullptr; + while (container != nullptr) { + // TODO: shall it be == instead? + if (container->hasContainerSize() && + container->containerSize() >= size && container->containerSize() <= size + 16) { + MEMORY_LOG("recycle %p for request %d\n", container, size) + result = container; + if (previous == nullptr) + state->finalizerQueue = container->nextLink(); + else + previous->setNextLink(container->nextLink()); + state->finalizerQueueSize--; + memset(container, 0, size); + break; + } + previous = container; + container = container->nextLink(); + } #endif - ContainerHeader* result = konanConstructSizedInstance(alignUp(size, kObjectAlignment)); + if (result == nullptr) { + result = konanConstructSizedInstance(alignUp(size, kObjectAlignment)); + atomicAdd(&allocCount, 1); + } CONTAINER_ALLOC_EVENT(state, size, result); #if TRACE_MEMORY state->containers->insert(result); #endif - atomicAdd(&allocCount, 1); return result; } ContainerHeader* AllocAggregatingFrozenContainer(KStdVector& containers) { auto componentSize = containers.size(); - auto* superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize); + auto* superContainer = AllocContainer(memoryState, sizeof(ContainerHeader) + sizeof(void*) * componentSize); auto* place = reinterpret_cast(superContainer + 1); for (auto* container : containers) { *place++ = container; @@ -1072,7 +1250,7 @@ void FreeAggregatingFrozenContainer(ContainerHeader* container) { MEMORY_LOG("Total subcontainers = %d\n", container->objectCount()); for (int i = 0; i < container->objectCount(); ++i) { MEMORY_LOG("Freeing subcontainer %p\n", *subContainer); - FreeContainer(*subContainer++); + freeContainer(*subContainer++); } #if USE_GC --state->finalizerQueueSuspendCount; @@ -1081,9 +1259,8 @@ void FreeAggregatingFrozenContainer(ContainerHeader* container) { MEMORY_LOG("Freeing subcontainers done\n"); } -void FreeContainer(ContainerHeader* container) { +void freeContainer(ContainerHeader* container) { RuntimeAssert(container != nullptr, "this kind of container shalln't be freed"); - auto state = memoryState; if (isAggregatingFrozenContainer(container)) { FreeAggregatingFrozenContainer(container); @@ -1093,49 +1270,44 @@ void FreeContainer(ContainerHeader* container) { runDeallocationHooks(container); // Now let's clean all object's fields in this container. - traverseContainerObjectFields(container, [](ObjHeader** location) { - ZeroHeapRef(location); + traverseContainerObjectFields(container, [container](ObjHeader** location) { + ZeroHeapRef(location); }); // And release underlying memory. if (isFreeable(container)) { container->setColorEvenIfGreen(CONTAINER_TAG_GC_BLACK); if (!container->buffered()) - scheduleDestroyContainer(state, container); + scheduleDestroyContainer(memoryState, container); } } -void ObjectContainer::Init(const TypeInfo* typeInfo) { +void ObjectContainer::Init(MemoryState* state, const TypeInfo* typeInfo) { RuntimeAssert(typeInfo->instanceSize_ >= 0, "Must be an object"); - uint32_t alloc_size = - sizeof(ContainerHeader) + typeInfo->instanceSize_; - header_ = AllocContainer(alloc_size); - if (header_) { - // One object in this container. - header_->setObjectCount(1); - // header->refCount_ is zero initialized by AllocContainer(). - SetHeader(GetPlace(), typeInfo); - MEMORY_LOG("object at %p\n", GetPlace()) - OBJECT_ALLOC_EVENT(memoryState, typeInfo->instanceSize_, GetPlace()) - } + uint32_t alloc_size = sizeof(ContainerHeader) + typeInfo->instanceSize_; + header_ = AllocContainer(state, alloc_size); + RuntimeCheck(header_ != nullptr, "Cannot alloc memory"); + // One object in this container, no need to set. + header_->setContainerSize(alloc_size); + RuntimeAssert(header_->objectCount() == 1, "Must work properly"); + // header->refCount_ is zero initialized by AllocContainer(). + SetHeader(GetPlace(), typeInfo); + OBJECT_ALLOC_EVENT(memoryState, typeInfo->instanceSize_, GetPlace()) } -void ArrayContainer::Init(const TypeInfo* typeInfo, uint32_t elements) { +void ArrayContainer::Init(MemoryState* state, const TypeInfo* typeInfo, uint32_t elements) { RuntimeAssert(typeInfo->instanceSize_ < 0, "Must be an array"); uint32_t alloc_size = sizeof(ContainerHeader) + arrayObjectSize(typeInfo, elements); - header_ = AllocContainer(alloc_size); - RuntimeAssert(header_ != nullptr, "Cannot alloc memory"); - if (header_) { - // One object in this container. - header_->setObjectCount(1); - // header->refCount_ is zero initialized by AllocContainer(). - GetPlace()->count_ = elements; - SetHeader(GetPlace()->obj(), typeInfo); - MEMORY_LOG("array at %p\n", GetPlace()) - OBJECT_ALLOC_EVENT( - memoryState, arrayObjectSize(typeInfo, elements), GetPlace()->obj()) - } + header_ = AllocContainer(state, alloc_size); + RuntimeCheck(header_ != nullptr, "Cannot alloc memory"); + // One object in this container, no need to set. + header_->setContainerSize(alloc_size); + RuntimeAssert(header_->objectCount() == 1, "Must work properly"); + // header->refCount_ is zero initialized by AllocContainer(). + GetPlace()->count_ = elements; + SetHeader(GetPlace()->obj(), typeInfo); + OBJECT_ALLOC_EVENT(memoryState, arrayObjectSize(typeInfo, elements), GetPlace()->obj()) } // TODO: store arena containers in some reuseable data structure, similar to @@ -1148,9 +1320,9 @@ void ArenaContainer::Deinit() { MEMORY_LOG("Arena::Deinit start: %p\n", this) auto chunk = currentChunk_; while (chunk != nullptr) { - // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set. + // freeContainer() doesn't release memory when CONTAINER_TAG_STACK is set. MEMORY_LOG("Arena::Deinit free chunk %p\n", chunk) - FreeContainer(chunk->asHeader()); + freeContainer(chunk->asHeader()); chunk = chunk->next; } chunk = currentChunk_; @@ -1166,7 +1338,7 @@ bool ArenaContainer::allocContainer(container_size_t minSize) { size = alignUp(size, kContainerAlignment); // TODO: keep simple cache of container chunks. ContainerChunk* result = konanConstructSizedInstance(size); - RuntimeAssert(result != nullptr, "Cannot alloc memory"); + RuntimeCheck(result != nullptr, "Cannot alloc memory"); if (result == nullptr) return false; result->next = currentChunk_; result->arena = this; @@ -1231,22 +1403,6 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun return result; } -inline void addRef(const ObjHeader* object) { - auto* container = object->container(); - if (container != nullptr) { - MEMORY_LOG("AddRef on %p in %p\n", object, container) - addRef(container); - } -} - -inline void releaseRef(const ObjHeader* object) { - auto* container = object->container(); - if (container != nullptr) { - MEMORY_LOG("ReleaseRef on %p in %p\n", object, container) - releaseRef(container); - } -} - void AddRefFromAssociatedObject(const ObjHeader* object) { AddHeapRef(const_cast(object)); } @@ -1255,6 +1411,148 @@ void ReleaseRefFromAssociatedObject(const ObjHeader* object) { ReleaseHeapRef(const_cast(object)); } +#if USE_GC +void incrementStack(MemoryState* state) { + FrameOverlay* frame = currentFrame; + while (frame != nullptr) { + ObjHeader** current = reinterpret_cast(frame + 1) + frame->parameters; + ObjHeader** end = current + frame->count - kFrameOverlaySlots - frame->parameters; + while (current < end) { + ObjHeader* obj = *current++; + if (obj != nullptr) { + auto* container = obj->container(); + if (container != nullptr && container->tag() == CONTAINER_TAG_NORMAL) + IncrementRC(container); + } + } + frame = frame->previous; + } +} + +void actualizeNewlyFrozenOnStack(MemoryState* state, const ContainerHeaderSet* newlyFrozen) { + // For all frozen objects in stack slots - perform reference increment. + FrameOverlay* frame = currentFrame; + MEMORY_LOG("actualizeNewlyFrozenOnStack: newly frozen size is %d\n", newlyFrozen->size()) + while (frame != nullptr) { + MEMORY_LOG("current frame %p: %d parameters %d locals\n", frame, frame->parameters, frame->count) + ObjHeader** current = reinterpret_cast(frame + 1) + frame->parameters; + ObjHeader** end = current + frame->count - kFrameOverlaySlots - frame->parameters; + while (current < end) { + ObjHeader* obj = *current; + current++; + if (obj != nullptr) { + auto* container = obj->container(); + // No need to use atomic increment yet, object is still local. + if (container != nullptr && container->frozen() && newlyFrozen->count(container) != 0) { + container->incRefCount(); + MEMORY_LOG("incremented rc of %p to %d\n", container, container->refCount()); + } + } + } + frame = frame->previous; + } + + // And actualize RC of those objects using toRelease set. + for (auto& container : *(state->toRelease)) { + if (!isMarkedAsRemoved(container) && container->frozen()) { + RuntimeAssert(newlyFrozen->count(container) != 0, "Must be newly frozen"); + // To account for aggregating containers. + ContainerHeader* realContainer = realFrozenContainer(container); + auto newRc = realContainer->decRefCount(); + MEMORY_LOG("decremented rc of %p to %d\n", realContainer, newRc); + container = markAsRemoved(container); + } + } +} + +void processDecrements(MemoryState* state) { + auto* toRelease = state->toRelease; + state->gcSuspendCount++; + while (toRelease->size() > 0) { + auto* container = toRelease->back(); + toRelease->pop_back(); + if (isMarkedAsRemoved(container)) + continue; + DecrementRC(container); + } + state->gcSuspendCount--; +} + +void decrementStack(MemoryState* state) { + state->gcSuspendCount++; + FrameOverlay* frame = currentFrame; + while (frame != nullptr) { + ObjHeader** current = reinterpret_cast(frame + 1) + frame->parameters; + ObjHeader** end = current + frame->count - kFrameOverlaySlots - frame->parameters; + while (current < end) { + ObjHeader* obj = *current++; + if (obj != nullptr) { + auto* container = obj->container(); + if (container != nullptr && container->tag() == CONTAINER_TAG_NORMAL) + EnqueueDecrementRC(container); + } + } + frame = frame->previous; + } + state->gcSuspendCount--; +} + +void garbageCollect(MemoryState* state, bool force) { + RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed"); + + GC_LOG(">>> %s GC: threshold = %d toFree %d toRelease %d\n", \ + force ? "forced" : "regular", state->gcThreshold, state->toFree->size(), state->toRelease->size()) + +#if GC_ERGONOMICS + auto gcStartTime = konan::getTimeMicros(); +#endif + + state->gcInProgress = true; + + incrementStack(state); + processDecrements(state); + size_t beforeDecrements = state->toRelease->size(); + decrementStack(state); + size_t afterDecrements = state->toRelease->size(); + ssize_t stackReferences = afterDecrements - beforeDecrements; + if (stackReferences * 5 > state->gcThreshold) { +#if GC_ERGONOMICS + increaseGcThreshold(state); + GC_LOG("||| GC: too many stack references, increased threshold to \n", state->gcThreshold); +#else + GC_LOG("Too many stack references for the threshold: %d vs %d\n", stackReferences, state->gcThreshold) +#endif + } + + GC_LOG("||| GC: toFree %d toRelease %d\n", state->toFree->size(), state->toRelease->size()) + + processFinalizerQueue(state); + + if (force || state->toFree->size() > kMaxToFreeSize) { + while (state->toFree->size() > 0) { + CollectCycles(state); + processFinalizerQueue(state); + } + } + + state->gcInProgress = false; + +#if GC_ERGONOMICS + auto gcEndTime = konan::getTimeMicros(); + auto gcToComputeRatio = double(gcEndTime - gcStartTime) / (gcStartTime - state->lastGcTimestamp + 1); + if (gcToComputeRatio > kGcToComputeRatioThreshold) { + increaseGcThreshold(state); + GC_LOG("Adjusting GC threshold to %d\n", state->gcThreshold); + } + GC_LOG("GC: duration=%lld sinceLast=%lld\n", (gcEndTime - gcStartTime), gcStartTime - state->lastGcTimestamp); + state->lastGcTimestamp = gcEndTime; +#endif + + GC_LOG("<<< GC: toFree %d toRelease %d\n", state->toFree->size(), state->toRelease->size()) +} + +#endif // USE_GC + extern "C" { MemoryState* InitMemory() { @@ -1274,8 +1572,9 @@ MemoryState* InitMemory() { memoryState->toFree = konanConstructInstance(); memoryState->roots = konanConstructInstance(); memoryState->gcInProgress = false; - initThreshold(memoryState, kGcThreshold); memoryState->gcSuspendCount = 0; + memoryState->toRelease = konanConstructInstance(); + initGcThreshold(memoryState, kGcThreshold); #endif atomicAdd(&aliveMemoryStatesCount, 1); return memoryState; @@ -1283,11 +1582,15 @@ MemoryState* InitMemory() { void DeinitMemory(MemoryState* memoryState) { #if USE_GC - GarbageCollect(); + do { + GC_LOG("Calling GarbageCollect from DeinitMemory()\n") + garbageCollect(memoryState, true); + } while (memoryState->toRelease->size() > 0); RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC"); + RuntimeAssert(memoryState->toRelease->size() == 0, "Some memory have not been released after GC"); konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->roots); - + konanDestructInstance(memoryState->toRelease); RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty"); RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty"); @@ -1327,36 +1630,36 @@ void ResumeMemory(MemoryState* state) { OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - if (isArenaSlot(OBJ_RESULT)) { - auto arena = initedArena(asArenaSlot(OBJ_RESULT)); - auto result = arena->PlaceObject(type_info); - MEMORY_LOG("instance %p in arena: %p\n", result, arena) - return result; + auto container = ObjectContainer(memoryState, type_info); + ContainerHeader* header = container.header(); + // We cannot collect until reference will be stored into the stack slot. + if (header->tag() == CONTAINER_TAG_NORMAL) { + IncrementRC(header); + EnqueueDecrementRC(header); } - RETURN_OBJ(ObjectContainer(type_info).GetPlace()); + RETURN_OBJ(container.GetPlace()); } OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); if (elements < 0) ThrowIllegalArgumentException(); - if (isArenaSlot(OBJ_RESULT)) { - auto arena = initedArena(asArenaSlot(OBJ_RESULT)); - auto result = arena->PlaceArray(type_info, elements)->obj(); - MEMORY_LOG("array[%d] %p in arena: %p\n", elements, result, arena) - return result; + auto container = ArrayContainer(memoryState, type_info, elements); + ContainerHeader* header = container.header(); + // We cannot collect until reference will be stored into the stack slot. + if (header->tag() == CONTAINER_TAG_NORMAL) { + IncrementRC(header); + EnqueueDecrementRC(header); } - RETURN_OBJ(ArrayContainer(type_info, elements).GetPlace()->obj()); + RETURN_OBJ(container.GetPlace()->obj()); } OBJ_GETTER(InitInstance, ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) { ObjHeader* value = *location; - if (value != nullptr) { // OK'ish, inited by someone else. RETURN_OBJ(value); } - ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); UpdateHeapRef(location, object); #if KONAN_NO_EXCEPTIONS @@ -1367,7 +1670,7 @@ OBJ_GETTER(InitInstance, ctor(object); return object; } catch (...) { - ZeroStackRef(OBJ_RESULT); + UpdateReturnRef(OBJ_RESULT, nullptr); ZeroHeapRef(location); throw; } @@ -1394,7 +1697,7 @@ OBJ_GETTER(InitSharedInstance, FreezeSubgraph(object); return object; } catch (...) { - ZeroStackRef(OBJ_RESULT); + UpdateReturnRef(OBJ_RESULT, nullptr); ZeroHeapRef(location); throw; } @@ -1428,7 +1731,7 @@ OBJ_GETTER(InitSharedInstance, synchronize(); return object; } catch (...) { - ZeroStackRef(OBJ_RESULT); + UpdateReturnRef(OBJ_RESULT, nullptr); ZeroHeapRef(location); ZeroHeapRef(localLocation); synchronize(); @@ -1438,12 +1741,20 @@ OBJ_GETTER(InitSharedInstance, #endif } +void SetStackRef(ObjHeader** location, const ObjHeader* object) { + MEMORY_LOG("SetStackRef *%p: %p\n", location, object) + UPDATE_REF_EVENT(memoryState, nullptr, object, location, 1); + if (object != nullptr) + AddStackRef(const_cast(object)); + *const_cast(location) = object; +} + void SetHeapRef(ObjHeader** location, const ObjHeader* object) { MEMORY_LOG("SetHeapRef *%p: %p\n", location, object) - UPDATE_REF_EVENT(memoryState, *location, object, location, 0); - *const_cast(location) = object; + UPDATE_REF_EVENT(memoryState, nullptr, object, location, 0); if (object != nullptr) AddHeapRef(const_cast(object)); + *const_cast(location) = object; } void ZeroHeapRef(ObjHeader** location) { @@ -1466,55 +1777,49 @@ void ZeroStackRef(ObjHeader** location) { } } -ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { - return isArenaSlot(returnSlot) ? returnSlot : localSlot; -} - -ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) { - if (param == nullptr) return localSlot; - auto container = param->container(); - if (container == nullptr || (container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK) - return localSlot; - auto chunk = reinterpret_cast(container) - 1; - return reinterpret_cast(reinterpret_cast(&chunk->arena) | ARENA_BIT); -} - -ALWAYS_INLINE void updateRef(ObjHeader** location, const ObjHeader* object) { - RuntimeAssert(!isArenaSlot(location), "must not be a slot"); +void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { + UPDATE_REF_EVENT(memoryState, *location, object, location, 1) + RuntimeAssert(object != reinterpret_cast(1), "Markers disallowed here"); ObjHeader* old = *location; if (old != object) { if (object != nullptr) { - addRef(object); + AddStackRef(object); } *const_cast(location) = object; - if (reinterpret_cast(old) > 1) { - releaseRef(old); + if (old != nullptr ) { + ReleaseStackRef(old); } } } -void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { - UPDATE_REF_EVENT(memoryState, *location, object, location, 1); - updateRef(location, object); -} - void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { UPDATE_REF_EVENT(memoryState, *location, object, location, 0); - updateRef(location, object); + ObjHeader* old = *location; + if (old != object) { + if (object != nullptr) { + AddHeapRef(object); + } + *const_cast(location) = object; + if (reinterpret_cast(old) > 1) { + ReleaseHeapRef(old); + } + } } -ALWAYS_INLINE inline ObjHeader** slotAddressFor(ObjHeader** returnSlot, const ObjHeader* value) { - if (!isArenaSlot(returnSlot)) return returnSlot; - // Not a subject of reference counting. - if (value == nullptr || !isRefCounted(value)) return nullptr; - return initedArena(asArenaSlot(returnSlot))->getSlot(); +ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { + RuntimeCheck(false, "No longer supported"); + return nullptr; } -ALWAYS_INLINE inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) { - returnSlot = slotAddressFor(returnSlot, value); - if (returnSlot == nullptr) return; +ObjHeader** GetParamSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { + RuntimeCheck(false, "No longer supported"); + return nullptr; +} + +inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) { + MEMORY_LOG("updateReturnRefAdded %p\n", returnSlot) ObjHeader* old = *returnSlot; - UPDATE_REF_EVENT(memoryState, old, value, returnSlot, 1); + UPDATE_REF_EVENT(memoryState, old, value, returnSlot, 1) *const_cast(returnSlot) = value; if (old != nullptr) { ReleaseStackRef(old); @@ -1522,8 +1827,6 @@ ALWAYS_INLINE inline void updateReturnRefAdded(ObjHeader** returnSlot, const Obj } void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* value) { - returnSlot = slotAddressFor(returnSlot, value); - if (returnSlot == nullptr) return; UpdateStackRef(returnSlot, value); } @@ -1548,87 +1851,48 @@ void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { } void EnterFrame(ObjHeader** start, int parameters, int count) { - MEMORY_LOG("EnterFrame %p .. %p\n", start, start + count + parameters) -} - -inline void releaseStackRefs(MemoryState* state, ObjHeader** start, int count) { - MEMORY_LOG("ReleaseStackRefs %p .. %p\n", start, start + count) - ObjHeader** current = start; - while (count-- > 0) { - ObjHeader* object = *current; - if (object != nullptr) { - UPDATE_REF_EVENT(memoryState, object, nullptr, current, 1); - ReleaseStackRef(object); - // Just for sanity, optional. - *current = nullptr; - } - current++; - } + MEMORY_LOG("EnterFrame %p: %d parameters %d locals\n", start, parameters, count) + FrameOverlay* frame = reinterpret_cast(start); + frame->previous = currentFrame; + currentFrame = frame; + // TODO: maybe compress in single value somehow. + frame->parameters = parameters; + frame->count = count; } void LeaveFrame(ObjHeader** start, int parameters, int count) { - MEMORY_LOG("LeaveFrame %p .. %p\n", start, start + count + parameters) - auto* state = memoryState; - releaseStackRefs(state, start + parameters + kFrameOverlaySlots, count - kFrameOverlaySlots - parameters); - if (*start != nullptr) { - auto arena = initedArena(start); - MEMORY_LOG("LeaveFrame: free arena %p\n", arena) - arena->Deinit(); - konanFreeMemory(arena); - MEMORY_LOG("LeaveFrame: free arena done %p\n", arena) + MEMORY_LOG("LeaveFrame %p: %d parameters %d locals\n", start, parameters, count) + MemoryState* state = memoryState; + ObjHeader** current = start + parameters + kFrameOverlaySlots; + ObjHeader** end = start + count; + while (current < end) { + ObjHeader* object = *current++; + if (object != nullptr) { + ReleaseStackRef(object); + } } + FrameOverlay* frame = reinterpret_cast(start); + currentFrame = frame->previous; } - #if USE_GC void GarbageCollect() { - MemoryState* state = memoryState; - RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed"); - - MEMORY_LOG("Garbage collect\n") - -#if GC_ERGONOMICS - auto gcStartTime = konan::getTimeMicros(); -#endif - - state->gcInProgress = true; - - processFinalizerQueue(state); - - while (state->toFree->size() > 0) { - CollectCycles(state); - processFinalizerQueue(state); - } - - state->gcInProgress = false; - -#if GC_ERGONOMICS - auto gcEndTime = konan::getTimeMicros(); - auto gcToComputeRatio = double(gcEndTime - gcStartTime) / (gcStartTime - state->lastGcTimestamp + 1); - if (gcToComputeRatio > kGcToComputeRatioThreshold) { - auto newThreshold = state->gcThreshold * 3 / 2 + 1; - if (newThreshold < kMaxErgonomicThreshold) { - MEMORY_LOG("Adjusting GC threshold to %d\n", newThreshold); - initThreshold(state, newThreshold); - } - } - MEMORY_LOG("Garbage collect: GC length=%lld sinceLast=%lld\n", - (gcEndTime - gcStartTime), gcStartTime - state->lastGcTimestamp); - state->lastGcTimestamp = gcEndTime; -#endif + garbageCollect(memoryState, true); } #endif // USE_GC void Kotlin_native_internal_GC_collect(KRef) { #if USE_GC + GC_LOG("Kotlin_native_internal_GC_collect\n") GarbageCollect(); #endif } void Kotlin_native_internal_GC_suspend(KRef) { #if USE_GC + GC_LOG("Kotlin_native_internal_GC_suspend\n") memoryState->gcSuspendCount++; #endif } @@ -1638,9 +1902,11 @@ void Kotlin_native_internal_GC_resume(KRef) { MemoryState* state = memoryState; if (state->gcSuspendCount > 0) { state->gcSuspendCount--; - if (state->toFree != nullptr && - freeableSize(state) >= state->gcThreshold) { - GarbageCollect(); + if (state->toRelease != nullptr && + state->toRelease->size() >= state->gcThreshold && + state->gcSuspendCount == 0) { + GC_LOG("Kotlin_native_internal_GC_resume\n") + garbageCollect(state, false); } } #endif @@ -1648,10 +1914,14 @@ void Kotlin_native_internal_GC_resume(KRef) { void Kotlin_native_internal_GC_stop(KRef) { #if USE_GC - if (memoryState->toFree != nullptr) { - GarbageCollect(); + GC_LOG("Kotlin_native_internal_GC_stop\n") + if (memoryState->toRelease != nullptr) { + memoryState->gcSuspendCount = 0; + garbageCollect(memoryState, true); + konanDestructInstance(memoryState->toRelease); konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->roots); + memoryState->toRelease = nullptr; memoryState->toFree = nullptr; memoryState->roots = nullptr; } @@ -1660,23 +1930,28 @@ void Kotlin_native_internal_GC_stop(KRef) { void Kotlin_native_internal_GC_start(KRef) { #if USE_GC + GC_LOG("Kotlin_native_internal_GC_start\n") if (memoryState->toFree == nullptr) { memoryState->toFree = konanConstructInstance(); + memoryState->toRelease = konanConstructInstance(); memoryState->roots = konanConstructInstance(); + memoryState->gcSuspendCount = 0; } #endif } void Kotlin_native_internal_GC_setThreshold(KRef, KInt value) { #if USE_GC + GC_LOG("Kotlin_native_internal_setThreshold %d\n", value) if (value > 0) { - initThreshold(memoryState, value); + initGcThreshold(memoryState, value); } #endif } KInt Kotlin_native_internal_GC_getThreshold(KRef) { #if USE_GC + GC_LOG("Kotlin_native_internal_getThreshold %d\n") return memoryState->gcThreshold; #else return -1; @@ -1685,6 +1960,7 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) { KNativePtr CreateStablePointer(KRef any) { if (any == nullptr) return nullptr; + MEMORY_LOG("CreateStablePointer for %p rc=%d\n", any, any->container() ? any->container()->refCount() : 0) AddHeapRef(any); return reinterpret_cast(any); } @@ -1703,10 +1979,10 @@ OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { OBJ_GETTER(AdoptStablePointer, KNativePtr pointer) { synchronize(); KRef ref = reinterpret_cast(pointer); - // TODO: as it is called from C runtime, - ZeroHeapRef(OBJ_RESULT); - // Somewhat hacky. - *OBJ_RESULT = ref; + MEMORY_LOG("adopting stable pointer %p, rc=%d\n", \ + ref, (ref && ref->container()) ? ref->container()->refCount() : -1) + UpdateReturnRef(OBJ_RESULT, ref); + DisposeStablePointer(pointer); return ref; } @@ -1718,7 +1994,10 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) { auto* container = toVisit.front(); toVisit.pop_front(); visited->insert(container); - if (container->refCount() != 0) return true; + if (container->refCount() > 0) { + MEMORY_LOG("container %p with rc %d blocks transfer\n", container, container->refCount()) + return true; + } traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) { auto* child = ref->container(); if (!Shareable(child) && (visited->count(child) == 0)) { @@ -1728,43 +2007,76 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) { } return false; } -#endif +#endif // USE_GC bool ClearSubgraphReferences(ObjHeader* root, bool checked) { #if USE_GC - if (root != nullptr) { - auto state = memoryState; - auto* container = root->container(); + MEMORY_LOG("ClearSubgraphReferences %p\n", root) + if (root == nullptr) return true; + auto state = memoryState; + auto* container = root->container(); - if (container == nullptr || container->frozen()) - // We assume, that frozen objects can be safely passed and are already removed - // GC candidate list. - return true; + if (Shareable(container)) + // We assume, that frozen/shareable objects can be safely passed and not present + // in the GC candidate list. + // TODO: assert for that? + return true; - ContainerHeaderSet visited; - if (!checked) { - hasExternalRefs(container, &visited); - } else { - if (!Shareable(container)) { - container->decRefCount(); - MarkGray(container); - auto bad = hasExternalRefs(container, &visited); - ScanBlack(container); - container->incRefCount(); - if (bad) return false; + ContainerHeaderSet visited; + if (!checked) { + hasExternalRefs(container, &visited); + } else { + // Now decrement RC of elements in toRelease set for reachibility analysis. + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto released = *it; + if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { + released->decRefCount(); } } - - // TODO: not very efficient traversal. - for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { - auto container = *it; - if (visited.count(container) != 0) { - container->resetBuffered(); - container->setColorAssertIfGreen(CONTAINER_TAG_GC_BLACK); - *it = markAsRemoved(container); - } + container->decRefCount(); + MarkGray(container); + auto bad = hasExternalRefs(container, &visited); + ScanBlack(container); + // Restore original RC. + container->incRefCount(); + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto released = *it; + if (!isMarkedAsRemoved(released) && released->tag() == CONTAINER_TAG_NORMAL) { + released->incRefCount(); + } + } + if (bad) { + return false; } } + + // Remove all no longer owned containers from GC structures. + // TODO: not very efficient traversal. + for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { + auto container = *it; + if (visited.count(container) != 0) { + MEMORY_LOG("removing %p from the toFree list\n", container) + container->resetBuffered(); + container->setColorAssertIfGreen(CONTAINER_TAG_GC_BLACK); + *it = markAsRemoved(container); + } + } + for (auto it = state->toRelease->begin(); it != state->toRelease->end(); ++it) { + auto container = *it; + if (!isMarkedAsRemoved(container) && visited.count(container) != 0) { + MEMORY_LOG("removing %p from the toRelease list\n", container) + container->decRefCount(); + *it = markAsRemoved(container); + } + } + +#if TRACE_MEMORY + // Forget transferred containers. + for (auto* it: visited) { + state->containers->erase(it); + } +#endif + #endif // USE_GC return true; } @@ -1840,7 +2152,7 @@ void traverseStronglyConnectedComponent(ContainerHeader* start, } } -void freezeAcyclic(ContainerHeader* rootContainer) { +void freezeAcyclic(ContainerHeader* rootContainer, ContainerHeaderSet* newlyFrozen) { KStdDeque queue; queue.push_back(rootContainer); while (!queue.empty()) { @@ -1851,6 +2163,9 @@ void freezeAcyclic(ContainerHeader* rootContainer) { current->setColorUnlessGreen(CONTAINER_TAG_GC_BLACK); // Note, that once object is frozen, it could be concurrently accessed, so // color and similar attributes shall not be used. + if (current->tag() == CONTAINER_TAG_NORMAL) + newlyFrozen->insert(current); + MEMORY_LOG("freezeing %p\n", current) current->freeze(); traverseContainerReferredObjects(current, [current, &queue](ObjHeader* obj) { ContainerHeader* objContainer = obj->container(); @@ -1862,7 +2177,9 @@ void freezeAcyclic(ContainerHeader* rootContainer) { } } -void freezeCyclic(ContainerHeader* rootContainer, const KStdVector& order) { +void freezeCyclic(ContainerHeader* rootContainer, + const KStdVector& order, + ContainerHeaderSet* newlyFrozen) { KStdUnorderedMap> reversedEdges; KStdDeque queue; queue.push_back(rootContainer); @@ -1915,8 +2232,11 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVectorresetBuffered(); container->setColorUnlessGreen(CONTAINER_TAG_GC_BLACK); + if (container->tag() == CONTAINER_TAG_NORMAL) + newlyFrozen->insert(container); // Note, that once object is frozen, it could be concurrently accessed, so // color and similar attributes shall not be used. + MEMORY_LOG("freezeing %p\n", container) container->freeze(); // We set refcount of original container to zero, so that it is seen as such after removal // meta-object, where aggregating container is stored. @@ -1925,7 +2245,10 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVectorsetRefCount(totalCount - internalRefsCount); + newlyFrozen->insert(superContainer); } } @@ -1968,11 +2291,12 @@ void FreezeSubgraph(ObjHeader* root) { if (firstBlocker != nullptr) { ThrowFreezingException(root, firstBlocker); } + ContainerHeaderSet newlyFrozen; // Now unmark all marked objects, and freeze them, if no cycles detected. if (hasCycles) { - freezeCyclic(rootContainer, order); + freezeCyclic(rootContainer, order, &newlyFrozen); } else { - freezeAcyclic(rootContainer ); + freezeAcyclic(rootContainer, &newlyFrozen); } #if USE_GC @@ -1981,9 +2305,13 @@ void FreezeSubgraph(ObjHeader* root) { // and use it when analyzing toFree during collection. auto state = memoryState; for (auto& container : *(state->toFree)) { - if (!isMarkedAsRemoved(container) && container->frozen()) - container = markAsRemoved(container); + if (!isMarkedAsRemoved(container) && container->frozen()) { + RuntimeAssert(newlyFrozen.count(container) != 0, "Must be newly frozen"); + container = markAsRemoved(container); + } } + // Actualize reference counters of newly frozen objects. + actualizeNewlyFrozenOnStack(memoryState, &newlyFrozen); #endif } @@ -2024,12 +2352,14 @@ void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlo ReleaseHeapRef(oldValue); } -OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) { +OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock) { + MEMORY_LOG("ReadHeapRefLocked: %p\n", location) lock(spinlock); ObjHeader* value = *location; // We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock. - if (value != nullptr) - AddHeapRef(value); + if (value != nullptr) { + AddStackRef(value); + } unlock(spinlock); updateReturnRefAdded(OBJ_RESULT, value); return value; diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index dc093b924d8..7aaa4d763fe 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -39,8 +39,9 @@ typedef enum { // Mask for container type. CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1, - // Shift to get actual object count. - CONTAINER_TAG_GC_SHIFT = 6, + // Shift to get actual object count, if has it. + CONTAINER_TAG_GC_SHIFT = 7, + CONTAINER_TAG_GC_MASK = (1 << CONTAINER_TAG_GC_SHIFT) - 1, CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT, // Color mask of a container. CONTAINER_TAG_COLOR_SHIFT = 3, @@ -64,7 +65,9 @@ typedef enum { // Individual state bits used during GC and freezing. CONTAINER_TAG_GC_MARKED = 1 << CONTAINER_TAG_COLOR_SHIFT, CONTAINER_TAG_GC_BUFFERED = 1 << (CONTAINER_TAG_COLOR_SHIFT + 1), - CONTAINER_TAG_GC_SEEN = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2) + CONTAINER_TAG_GC_SEEN = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2), + // If indeed has more that one object. + CONTAINER_TAG_GC_HAS_OBJECT_COUNT = 1 << (CONTAINER_TAG_COLOR_SHIFT + 3) } ContainerTag; typedef enum { @@ -109,8 +112,8 @@ struct ContainerHeader { return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK; } - inline unsigned refCount() const { - return refCount_ >> CONTAINER_TAG_SHIFT; + inline int refCount() const { + return (int)refCount_ >> CONTAINER_TAG_SHIFT; } inline void setRefCount(unsigned refCount) { @@ -140,20 +143,50 @@ struct ContainerHeader { return value >> CONTAINER_TAG_SHIFT; } + inline int decRefCount() { + #ifdef KONAN_NO_THREADS + int value = refCount_ -= CONTAINER_TAG_INCREMENT; + #else + int value = shareable() ? + __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT; + #endif + return value >> CONTAINER_TAG_SHIFT; + } + inline unsigned tag() const { return refCount_ & CONTAINER_TAG_MASK; } inline unsigned objectCount() const { - return objectCount_ >> CONTAINER_TAG_GC_SHIFT; + return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0 ? + (objectCount_ >> CONTAINER_TAG_GC_SHIFT) : 1; } inline void incObjectCount() { + RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0, "Must have object count"); objectCount_ += CONTAINER_TAG_GC_INCREMENT; } inline void setObjectCount(int count) { - objectCount_ = count << CONTAINER_TAG_GC_SHIFT; + if (count == 1) { + objectCount_ &= ~CONTAINER_TAG_GC_HAS_OBJECT_COUNT; + } else { + objectCount_ = (count << CONTAINER_TAG_GC_SHIFT) | CONTAINER_TAG_GC_HAS_OBJECT_COUNT; + } + } + + inline unsigned containerSize() const { + RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must be single-object"); + return (objectCount_ >> CONTAINER_TAG_GC_SHIFT); + } + + inline void setContainerSize(unsigned size) { + RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must not have object count"); + objectCount_ = (objectCount_ & CONTAINER_TAG_GC_MASK) | (size << CONTAINER_TAG_GC_SHIFT); + } + + inline bool hasContainerSize() { + return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0; } inline unsigned color() const { @@ -213,6 +246,7 @@ struct ContainerHeader { objectCount_ &= ~CONTAINER_TAG_GC_SEEN; } + // Following operations only work on freed container which is in finalization queue. // We cannot use 'this' here, as it conflicts with aliasing analysis in clang. inline void setNextLink(ContainerHeader* next) { *reinterpret_cast(this + 1) = next; @@ -296,10 +330,11 @@ struct ObjHeader { ContainerHeader* container() const { unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK); - if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) return nullptr; - return (bits & OBJECT_TAG_NONTRIVIAL_CONTAINER) != 0 ? - (reinterpret_cast(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_ : - reinterpret_cast(const_cast(this)) - 1; + if ((bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) == 0) + return reinterpret_cast(const_cast(this)) - 1; + if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) + return nullptr; + return (reinterpret_cast(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_; } // Unsafe cast to ArrayHeader. Use carefully! @@ -336,6 +371,8 @@ inline bool PermanentOrFrozen(ObjHeader* obj) { // Class representing arbitrary placement container. class Container { + public: + ContainerHeader* header() const { return header_; } protected: // Data where everything is being stored. ContainerHeader* header_; @@ -350,97 +387,10 @@ class Container { } }; -// Container for a single object. -class ObjectContainer : public Container { - public: - // Single instance. - explicit ObjectContainer(const TypeInfo* type_info) { - Init(type_info); - } - - // Object container shalln't have any dtor, as it's being freed by - // ::Release(). - - ObjHeader* GetPlace() const { - return reinterpret_cast(header_ + 1); - } - - private: - void Init(const TypeInfo* type_info); -}; - - -class ArrayContainer : public Container { - public: - ArrayContainer(const TypeInfo* type_info, uint32_t elements) { - Init(type_info, elements); - } - - // Array container shalln't have any dtor, as it's being freed by ::Release(). - - ArrayHeader* GetPlace() const { - return reinterpret_cast(header_ + 1); - } - - private: - void Init(const TypeInfo* type_info, uint32_t elements); -}; - -// Class representing arena-style placement container. -// Container is used for reference counting, and it is assumed that objects -// with related placement will share container. Only -// whole container can be freed, individual objects are not taken into account. -class ArenaContainer; - -struct ContainerChunk { - ContainerChunk* next; - ArenaContainer* arena; - // Then we have ContainerHeader here. - ContainerHeader* asHeader() { - return reinterpret_cast(this + 1); - } -}; - -class ArenaContainer { - public: - void Init(); - void Deinit(); - - // Place individual object in this container. - ObjHeader* PlaceObject(const TypeInfo* type_info); - - // Places an array of certain type in this container. Note that array_type_info - // is type info for an array, not for an individual element. Also note that exactly - // same operation could be used to place strings. - ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count); - - ObjHeader** getSlot(); - - private: - void* place(container_size_t size); - - bool allocContainer(container_size_t minSize); - - void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) { - obj->typeInfoOrMeta_ = const_cast(typeInfo); - obj->setContainer(currentChunk_->asHeader()); - // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC. - } - - ContainerChunk* currentChunk_; - uint8_t* current_; - uint8_t* end_; - ArrayHeader* slots_; - uint32_t slotsCount_; -}; - #ifdef __cplusplus extern "C" { #endif -// Bit or'ed to slot pointer, marking the fact that allocation shall happen -// in arena pointed by the slot. -#define ARENA_BIT 1 #define OBJ_RESULT __result__ #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT) #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT) @@ -508,6 +458,8 @@ void WeakReferenceCounterClear(ObjHeader* counter); // in intermediate frames when throwing // +// Sets stack location. +void SetStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Sets heap location. void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Zeroes heap location. @@ -528,15 +480,11 @@ OBJ_GETTER(SwapHeapRefLocked, // Sets reference with taken lock. void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW; // Reads reference with taken lock. -OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW; +OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW; // Called on frame enter, if it has object slots. void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; // Called on frame leave, if it has object slots. void LeaveFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; -// Tries to use returnSlot's arena for allocation. -ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) RUNTIME_NOTHROW; -// Tries to use param's arena for allocation. -ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) RUNTIME_NOTHROW; // Collect garbage, which cannot be found by reference counting (cycles). void GarbageCollect() RUNTIME_NOTHROW; // Clears object subgraph references from memory subsystem, and optionally @@ -561,27 +509,67 @@ void EnsureNeverFrozen(ObjHeader* obj); } #endif + +struct FrameOverlay { + void* arena; + FrameOverlay* previous; + // As they go in pair, sizeof(FrameOverlay) % sizeof(void*) == 0 is always held. + int32_t parameters; + int32_t count; +}; + // Class holding reference to an object, holding object during C++ scope. class ObjHolder { public: - ObjHolder() : obj_(nullptr) {} + ObjHolder() : obj_(nullptr) { + EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*)); + } explicit ObjHolder(const ObjHeader* obj) { - ::SetHeapRef(&obj_, obj); + EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*)); + ::SetStackRef(slot(), obj); } + ~ObjHolder() { - ::ZeroHeapRef(&obj_); + LeaveFrame(frame(), 0, sizeof(*this)/sizeof(void*)); } ObjHeader* obj() { return obj_; } - const ObjHeader* obj() const { return obj_; } - ObjHeader** slot() { return &obj_; } - void clear() { ::ZeroHeapRef(&obj_); } - private: + const ObjHeader* obj() const { return obj_; } + + ObjHeader** slot() { + return &obj_; + } + + void clear() { ::ZeroStackRef(&obj_); } + + private: + ObjHeader** frame() { return reinterpret_cast(&frame_); } + + FrameOverlay frame_; ObjHeader* obj_; }; +class ExceptionObjHolder { + public: + explicit ExceptionObjHolder(const ObjHeader* obj) { + ::SetHeapRef(&obj_, obj); + } + + ~ExceptionObjHolder() { + ZeroHeapRef(&obj_); + } + + ObjHeader* obj() { return obj_; } + + const ObjHeader* obj() const { return obj_; } + + private: + ObjHeader* obj_; +}; + + class KRefSharedHolder { public: inline ObjHeader** slotToInit() { diff --git a/runtime/src/main/cpp/Worker.cpp b/runtime/src/main/cpp/Worker.cpp index 5d5ccf3cbdd..e6fb7a08dba 100644 --- a/runtime/src/main/cpp/Worker.cpp +++ b/runtime/src/main/cpp/Worker.cpp @@ -61,18 +61,14 @@ enum { THREAD_LOCAL_VARIABLE KInt g_currentWorkerId = 0; -KNativePtr transfer(KRef object, KInt mode) { - switch (mode) { - case CHECKED: - case UNCHECKED: - if (!ClearSubgraphReferences(object, mode == CHECKED)) { - // Release reference to the object, as it is not being managed by ObjHolder. - ZeroHeapRef(&object); - ThrowWorkerInvalidState(); - } - return object; +KNativePtr transfer(ObjHolder* holder, KInt mode) { + void* result = CreateStablePointer(holder->obj()); + if (!ClearSubgraphReferences(holder->obj(), mode == CHECKED)) { + DisposeStablePointer(result); + ThrowWorkerInvalidState(); } - return nullptr; + holder->clear(); + return result; } class Locker { @@ -373,34 +369,33 @@ void* workerRoutine(void* argument) { g_currentWorkerId = worker->id(); Kotlin_initRuntimeIfNeeded(); - while (true) { - Job job = worker->getJob(); - if (job.function == nullptr) { - // Termination request, notify the future. - job.future->storeResultUnlocked(nullptr, true); - theState()->removeWorkerUnlocked(worker->id()); - break; - } + { ObjHolder argumentHolder; - KRef argument = AdoptStablePointer(job.argument, argumentHolder.slot()); - // Note that this is a bit hacky, as we must not auto-release resultRef, - // so we don't use ObjHolder. - // It is so, as ownership is transferred. - KRef resultRef = nullptr; - KNativePtr result = nullptr; - bool ok = true; - try { - job.function(argument, &resultRef); + ObjHolder resultHolder; + while (true) { + Job job = worker->getJob(); + if (job.function == nullptr) { + // Termination request, notify the future. + job.future->storeResultUnlocked(nullptr, true); + theState()->removeWorkerUnlocked(worker->id()); + break; + } + KRef argument = AdoptStablePointer(job.argument, argumentHolder.slot()); + KNativePtr result = nullptr; + bool ok = true; + try { + job.function(argument, resultHolder.slot()); argumentHolder.clear(); // Transfer the result. - result = transfer(resultRef, job.transferMode); - } catch (ObjHolder& e) { + result = transfer(&resultHolder, job.transferMode); + } catch (ExceptionObjHolder& e) { ok = false; if (worker->errorReporting()) - ReportUnhandledException(e.obj()); + ReportUnhandledException(e.obj()); + } + // Notify the future. + job.future->storeResultUnlocked(result, ok); } - // Notify the future. - job.future->storeResultUnlocked(result, ok); } Kotlin_deinitRuntimeIfNeeded(); @@ -424,11 +419,9 @@ KInt currentWorker() { KInt schedule(KInt id, KInt transferMode, KRef producer, KNativePtr jobFunction) { Job job; - // Note that this is a bit hacky, as we must not auto-release jobArgumentRef, - // so we don't use ObjHolder. - KRef jobArgumentRef = nullptr; - WorkerLaunchpad(producer, &jobArgumentRef); - KNativePtr jobArgument = transfer(jobArgumentRef, transferMode); + ObjHolder holder; + WorkerLaunchpad(producer, holder.slot()); + KNativePtr jobArgument = transfer(&holder, transferMode); Future* future = theState()->addJobToWorkerUnlocked(id, jobFunction, jobArgument, false, transferMode); if (future == nullptr) ThrowWorkerInvalidState(); return future->id(); @@ -462,12 +455,13 @@ OBJ_GETTER(attachObjectGraphInternal, KNativePtr stable) { } KNativePtr detachObjectGraphInternal(KInt transferMode, KRef producer) { - KRef ref = nullptr; - WorkerLaunchpad(producer, &ref); - if (ref != nullptr) { - return transfer(ref, transferMode); - } else + ObjHolder result; + WorkerLaunchpad(producer, result.slot()); + if (result.obj() != nullptr) { + return transfer(&result, transferMode); + } else { return nullptr; + } } #else diff --git a/runtime/src/main/kotlin/kotlin/native/Annotations.kt b/runtime/src/main/kotlin/kotlin/native/Annotations.kt index 3e9babe4fc7..18bf26239cc 100644 --- a/runtime/src/main/kotlin/kotlin/native/Annotations.kt +++ b/runtime/src/main/kotlin/kotlin/native/Annotations.kt @@ -24,6 +24,14 @@ public annotation class SymbolName(val name: String) @Retention(AnnotationRetention.BINARY) public annotation class Retain +/** + * Preserve the function entry point during global optimizations, only for the given target. + */ +@Target(AnnotationTarget.FUNCTION, AnnotationTarget.CLASS) +@Retention(AnnotationRetention.BINARY) +public annotation class RetainForTarget(val target: String) + + // TODO: merge with [kotlin.jvm.Throws] /** * This annotation indicates what exceptions should be declared by a function when compiled to a platform method.