From c3b2b47403b6705e25ed6744615a0c93a21a153c Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Fri, 22 Mar 2019 15:50:42 +0300 Subject: [PATCH] Distinguish stack/heap writes and collect stats. (#2794) --- .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 1 + .../cli/bc/K2NativeCompilerArguments.kt | 3 + .../kotlin/backend/konan/CAdapterGenerator.kt | 6 +- .../kotlin/backend/konan/TestRunnerKind.kt | 3 +- .../backend/konan/llvm/CodeGenerator.kt | 18 +- .../kotlin/backend/konan/llvm/ContextUtils.kt | 3 +- .../kotlin/backend/konan/llvm/EntryPoint.kt | 1 + .../kotlin/backend/konan/llvm/IrToBitcode.kt | 16 +- .../backend/konan/llvm/VariableManager.kt | 6 +- .../llvm/objcexport/BlockPointerSupport.kt | 10 +- runtime/src/main/cpp/Arrays.cpp | 8 +- runtime/src/main/cpp/Atomic.cpp | 6 +- runtime/src/main/cpp/Atomic.h | 6 + runtime/src/main/cpp/Exceptions.cpp | 4 +- runtime/src/main/cpp/Memory.cpp | 401 +++++++++++------- runtime/src/main/cpp/Memory.h | 34 +- runtime/src/main/cpp/Weak.cpp | 2 +- runtime/src/main/cpp/Worker.cpp | 3 +- .../kotlin/native/internal/test/Launcher.kt | 4 + 19 files changed, 325 insertions(+), 210 deletions(-) diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 313ae273993..88e59df6dd1 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -181,6 +181,7 @@ class K2Native : CLICompiler() { when { arguments.generateWorkerTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.WORKER) arguments.generateTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.MAIN_THREAD) + arguments.generateNoExitTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.MAIN_THREAD_NO_EXIT) else -> put(GENERATE_TEST_RUNNER, TestRunnerKind.NONE) } // We need to download dependencies only if we use them ( = there are files to compile). diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt index dd38cca08de..3927a073fdd 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt @@ -29,6 +29,9 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { @Argument(value = "-generate-worker-test-runner", shortName = "-trw", description = "Produce a worker runner for unit tests") var generateWorkerTestRunner = false + @Argument(value = "-generate-no-exit-test-runner", + shortName = "-trn", description = "Produce a runner for unit tests not forcing exit") + var generateNoExitTestRunner = false @Argument(value="-include-binary", deprecatedName = "-includeBinary", shortName = "-ib", valueDescription = "", description = "Pack external binary within the klib") var includeBinaries: Array? = null diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt index ee04df669c4..2ff025811c3 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt @@ -849,7 +849,7 @@ internal class CAdapterGenerator( |#define RUNTIME_USED __attribute__((used)) | |extern "C" { - |void UpdateRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW; + |void UpdateHeapRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW; |KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW; |KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW; |void* CreateStablePointer(KObjHeader*) RUNTIME_NOTHROW; @@ -866,10 +866,10 @@ internal class CAdapterGenerator( |public: | KObjHolder() : obj_(nullptr) {} | explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) { - | UpdateRef(&obj_, obj); + | UpdateHeapRef(&obj_, obj); | } | ~KObjHolder() { - | UpdateRef(&obj_, nullptr); + | UpdateHeapRef(&obj_, nullptr); | } | KObjHeader* obj() { return obj_; } | KObjHeader** slot() { return &obj_; } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/TestRunnerKind.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/TestRunnerKind.kt index 06b264b5498..9abad940da5 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/TestRunnerKind.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/TestRunnerKind.kt @@ -9,5 +9,6 @@ package org.jetbrains.kotlin.backend.konan enum class TestRunnerKind { NONE, MAIN_THREAD, - WORKER + WORKER, + MAIN_THREAD_NO_EXIT } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 3447e2a9080..ce9edbff141 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -270,7 +270,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, val value = LLVMBuildLoad(builder, address, name)!! if (isObjectRef(value) && isVar) { val slot = alloca(LLVMTypeOf(value), variableLocation = null) - storeAny(value, slot) + storeStackRef(value, slot) } return value } @@ -281,9 +281,17 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, LLVMBuildStore(builder, value, ptr) } - fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef) { + fun storeHeapRef(value: LLVMValueRef, ptr: LLVMValueRef) { + updateRef(value, ptr, onStack = false) + } + + fun storeStackRef(value: LLVMValueRef, ptr: LLVMValueRef) { + updateRef(value, ptr, onStack = true) + } + + fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean) { if (isObjectRef(value)) { - updateRef(value, ptr) + if (onStack) storeStackRef(value, ptr) else storeHeapRef(value, ptr) } else { LLVMBuildStore(builder, value, ptr) } @@ -302,8 +310,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, call(context.llvm.updateReturnRefFunction, listOf(address, value)) } - private fun updateRef(value: LLVMValueRef, address: LLVMValueRef) { - call(context.llvm.updateRefFunction, listOf(address, value)) + private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean) { + call(if (onStack) context.llvm.updateStackRefFunction else context.llvm.updateHeapRefFunction, listOf(address, value)) } //-------------------------------------------------------------------------// diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index c521ddb58bc..63def798176 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -401,7 +401,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val initInstanceFunction = importRtFunction("InitInstance") val initSharedInstanceFunction = importRtFunction("InitSharedInstance") val updateReturnRefFunction = importRtFunction("UpdateReturnRef") - val updateRefFunction = importRtFunction("UpdateRef") + val updateHeapRefFunction = importRtFunction("UpdateHeapRef") + val updateStackRefFunction = importRtFunction("UpdateStackRef") val enterFrameFunction = importRtFunction("EnterFrame") val leaveFrameFunction = importRtFunction("LeaveFrame") val getReturnSlotIfArenaFunction = importRtFunction("GetReturnSlotIfArena") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EntryPoint.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EntryPoint.kt index ad6b087ee15..8370ff23a62 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EntryPoint.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EntryPoint.kt @@ -51,6 +51,7 @@ private fun defaultEntryName(config: CompilerConfiguration): String = when (config.get(KonanConfigKeys.GENERATE_TEST_RUNNER)) { TestRunnerKind.MAIN_THREAD -> "kotlin.native.internal.test.main" TestRunnerKind.WORKER -> "kotlin.native.internal.test.worker" + TestRunnerKind.MAIN_THREAD_NO_EXIT -> "kotlin.native.internal.test.mainNoExit" else -> "main" } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index b2503565411..4d3dbfbe4f2 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -371,7 +371,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map - storeAny(value, structGep(blockOnStackBase, index)) + // Although value is actually on the stack, it's not in normal slot area, so we cannot handle it + // as if it was on the stack. + store(value, structGep(blockOnStackBase, index)) } // Note: it is the slot in the block located on stack, so no need to manage it properly: diff --git a/runtime/src/main/cpp/Arrays.cpp b/runtime/src/main/cpp/Arrays.cpp index 70cc31bb101..239cadfc236 100644 --- a/runtime/src/main/cpp/Arrays.cpp +++ b/runtime/src/main/cpp/Arrays.cpp @@ -92,7 +92,7 @@ void Kotlin_Array_set(KRef thiz, KInt index, KConstRef value) { ThrowArrayIndexOutOfBoundsException(); } mutabilityCheck(thiz); - UpdateRef(ArrayAddressOfElementAt(array, index), value); + UpdateHeapRef(ArrayAddressOfElementAt(array, index), value); } KInt Kotlin_Array_getArrayLength(KConstRef thiz) { @@ -107,7 +107,7 @@ void Kotlin_Array_fillImpl(KRef thiz, KInt fromIndex, KInt toIndex, KRef value) } mutabilityCheck(thiz); for (KInt index = fromIndex; index < toIndex; ++index) { - UpdateRef(ArrayAddressOfElementAt(array, index), value); + UpdateHeapRef(ArrayAddressOfElementAt(array, index), value); } } @@ -123,12 +123,12 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex, mutabilityCheck(destination); if (fromIndex >= toIndex) { for (int index = 0; index < count; index++) { - UpdateRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), *ArrayAddressOfElementAt(array, fromIndex + index)); } } else { for (int index = count - 1; index >= 0; index--) { - UpdateRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), *ArrayAddressOfElementAt(array, fromIndex + index)); } } diff --git a/runtime/src/main/cpp/Atomic.cpp b/runtime/src/main/cpp/Atomic.cpp index 3a01721f5f1..1ee6cd5b6d9 100644 --- a/runtime/src/main/cpp/Atomic.cpp +++ b/runtime/src/main/cpp/Atomic.cpp @@ -184,7 +184,7 @@ OBJ_GETTER(Kotlin_AtomicReference_compareAndSwap, KRef thiz, KRef expectedValue, Kotlin_AtomicReference_checkIfFrozen(newValue); // See Kotlin_AtomicReference_get() for explanations, why locking is needed. AtomicReferenceLayout* ref = asAtomicReference(thiz); - RETURN_RESULT_OF(SwapRefLocked, &ref->value_, expectedValue, newValue, &ref->lock_); + RETURN_RESULT_OF(SwapHeapRefLocked, &ref->value_, expectedValue, newValue, &ref->lock_); } KBoolean Kotlin_AtomicReference_compareAndSet(KRef thiz, KRef expectedValue, KRef newValue) { @@ -192,14 +192,14 @@ KBoolean Kotlin_AtomicReference_compareAndSet(KRef thiz, KRef expectedValue, KRe // See Kotlin_AtomicReference_get() for explanations, why locking is needed. AtomicReferenceLayout* ref = asAtomicReference(thiz); ObjHolder holder; - auto old = SwapRefLocked(&ref->value_, expectedValue, newValue, &ref->lock_, holder.slot()); + auto old = SwapHeapRefLocked(&ref->value_, expectedValue, newValue, &ref->lock_, holder.slot()); return old == expectedValue; } void Kotlin_AtomicReference_set(KRef thiz, KRef newValue) { Kotlin_AtomicReference_checkIfFrozen(newValue); AtomicReferenceLayout* ref = asAtomicReference(thiz); - SetRefLocked(&ref->value_, newValue, &ref->lock_); + SetHeapRefLocked(&ref->value_, newValue, &ref->lock_); } OBJ_GETTER(Kotlin_AtomicReference_get, KRef thiz) { diff --git a/runtime/src/main/cpp/Atomic.h b/runtime/src/main/cpp/Atomic.h index 26e25289a6d..2240fedd329 100644 --- a/runtime/src/main/cpp/Atomic.h +++ b/runtime/src/main/cpp/Atomic.h @@ -60,4 +60,10 @@ ALWAYS_INLINE inline T atomicGet(volatile T* where) { #endif } +static ALWAYS_INLINE inline void synchronize() { +#ifndef KONAN_NO_THREADS + __sync_synchronize(); +#endif +} + #endif // RUNTIME_ATOMIC_H \ No newline at end of file diff --git a/runtime/src/main/cpp/Exceptions.cpp b/runtime/src/main/cpp/Exceptions.cpp index 6b51173ac1a..c903d5809ab 100644 --- a/runtime/src/main/cpp/Exceptions.cpp +++ b/runtime/src/main/cpp/Exceptions.cpp @@ -215,13 +215,13 @@ void ThrowException(KRef exception) { } OBJ_GETTER(Kotlin_setUnhandledExceptionHook, KRef hook) { - RETURN_RESULT_OF(SwapRefLocked, + RETURN_RESULT_OF(SwapHeapRefLocked, ¤tUnhandledExceptionHook, currentUnhandledExceptionHook, hook, ¤tUnhandledExceptionHookLock); } void OnUnhandledException(KRef throwable) { ObjHolder handlerHolder; - auto* handler = SwapRefLocked(¤tUnhandledExceptionHook, currentUnhandledExceptionHook, nullptr, + auto* handler = SwapHeapRefLocked(¤tUnhandledExceptionHook, currentUnhandledExceptionHook, nullptr, ¤tUnhandledExceptionHookLock, handlerHolder.slot()); if (handler == nullptr) { ReportUnhandledException(throwable); diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 4cbc1e3cbb3..ff3595fbb3a 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -92,8 +92,8 @@ struct FrameOverlay { FrameOverlay exportFrameOverlay; // Current number of allocated containers. -int allocCount = 0; -int aliveMemoryStatesCount = 0; +volatile int allocCount = 0; +volatile int aliveMemoryStatesCount = 0; // Forward declarations. void FreeContainer(ContainerHeader* header); @@ -102,11 +102,11 @@ void FreeContainer(ContainerHeader* header); class MemoryStatistic { public: // UpdateRef per-object type counters. - uint64_t updateCounters[5][5]; + uint64_t updateCounters[10][10]; // Alloc per container type counters. - uint64_t containerAllocs[5][2]; + uint64_t containerAllocs[2]; // Free per container type counters. - uint64_t objectAllocs[5][2]; + uint64_t objectAllocs[5]; // Histogram of allocation size distribution. KStdUnorderedMap* allocationHistogram; // Number of allocation cache hits. @@ -142,11 +142,11 @@ public: allocationHistogram = nullptr; } - void incAddRef(const ContainerHeader* header, bool atomic) { + void incAddRef(const ContainerHeader* header, bool atomic, int stack) { if (atomic) atomicAddRefs++; else addRefs++; } - void incReleaseRef(const ContainerHeader* header, bool atomic, bool cyclic) { + void incReleaseRef(const ContainerHeader* header, bool atomic, bool cyclic, int stack) { if (atomic) { RuntimeAssert(!cyclic, "Atomic updates cannot be cyclic yet"); atomicReleaseRefs++; @@ -155,69 +155,89 @@ public: } } - void incUpdateRef(const ObjHeader* objOld, const ObjHeader* objNew) { - updateCounters[toIndex(objOld)][toIndex(objNew)]++; + void incUpdateRef(const ObjHeader* objOld, const ObjHeader* objNew, int stack) { + updateCounters[toIndex(objOld, stack)][toIndex(objNew, stack)]++; } void incAlloc(size_t size, const ContainerHeader* header) { - containerAllocs[toIndex(header)][0]++; + containerAllocs[0]++; ++(*allocationHistogram)[size]; } void incFree(const ContainerHeader* header) { - containerAllocs[toIndex(header)][1]++; + containerAllocs[1]++; } void incAlloc(size_t size, const ObjHeader* header) { - objectAllocs[toIndex(header)][0]++; + objectAllocs[toIndex(header, 0)]++; } - void incFree(const ObjHeader* header) { - objectAllocs[toIndex(header)][1]++; - } - - static int toIndex(const ObjHeader* obj) { + static int toIndex(const ObjHeader* obj, int stack) { if (reinterpret_cast(obj) > 1) - return toIndex(obj->container()); + return toIndex(obj->container(), stack); else - return 4; + return 4 + stack * 5; } - static int toIndex(const ContainerHeader* header) { - if (header == nullptr) return 2; // permanent. + static int toIndex(const ContainerHeader* header, int stack) { + if (header == nullptr) return 2 + stack * 5; // permanent. switch (header->tag()) { - case CONTAINER_TAG_NORMAL : return 0; - case CONTAINER_TAG_STACK : return 1; - case CONTAINER_TAG_FROZEN: return 3; + case CONTAINER_TAG_NORMAL : return 0 + stack * 5; + case CONTAINER_TAG_STACK : return 1 + stack * 5; + case CONTAINER_TAG_FROZEN: return 3 + stack * 5; } RuntimeAssert(false, "unknown container type"); return -1; } static double percents(uint64_t value, uint64_t all) { - return ((double)value / (double)all) * 100.0; + return all == 0 ? 0 : ((double)value / (double)all) * 100.0; } void printStatistic() { konan::consolePrintf("\nMemory manager statistic:\n\n"); - for (int i = 0; i < 2; i++) { - konan::consolePrintf("Container %s alloc: %lld, free: %lld\n", - indexToName[i], containerAllocs[i][0], - containerAllocs[i][1]); + konan::consolePrintf("Container alloc: %lld, free: %lld\n", + containerAllocs[0], containerAllocs[1]); + for (int i = 0; i < 5; i++) { + // Only normal and frozen can be allocated. + if (i == 0 || i == 3) + konan::consolePrintf("Object %s alloc: %lld\n", indexToName[i], objectAllocs[i]); } - for (int i = 0; i < 2; i++) { - konan::consolePrintf("Object %s alloc: %lld, free: %lld\n", - indexToName[i], objectAllocs[i][0], - objectAllocs[i][1]); - } - konan::consolePrintf("\n"); + + uint64_t allUpdateRefs = 0, heapUpdateRefs = 0, stackUpdateRefs = 0; + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + allUpdateRefs += updateCounters[i][j]; + if (i < 5 && j < 5) + heapUpdateRefs += updateCounters[i][j]; + if (i >= 5 && j >= 5) + stackUpdateRefs += updateCounters[i][j]; + } + } + konan::consolePrintf("Total updates: %lld, stack: %lld(%.2lf%%), heap: %lld(%.2lf%%)\n", + allUpdateRefs, + stackUpdateRefs, percents(stackUpdateRefs, allUpdateRefs), + heapUpdateRefs, percents(heapUpdateRefs, allUpdateRefs)); for (int i = 0; i < 5; i++) { for (int j = 0; j < 5; j++) { - konan::consolePrintf("UpdateRef[%s -> %s]: %lld\n", - indexToName[i], indexToName[j], updateCounters[i][j]); + if (updateCounters[i][j] != 0) + konan::consolePrintf("UpdateHeapRef[%s -> %s]: %lld (%.2lf%% of all, %.2lf%% of heap)\n", + indexToName[i], indexToName[j], updateCounters[i][j], + percents(updateCounters[i][j], allUpdateRefs), + percents(updateCounters[i][j], heapUpdateRefs)); } } + for (int i = 5; i < 10; i++) { + for (int j = 5; j < 10; j++) { + if (updateCounters[i][j] != 0) + konan::consolePrintf("UpdateStackRef[%s -> %s]: %lld (%.2lf%% of all, %.2lf%% of stack)\n", + indexToName[i - 5], indexToName[j - 5], + updateCounters[i][j], + percents(updateCounters[i][j], allUpdateRefs), + percents(updateCounters[i][j], stackUpdateRefs)); + } + } konan::consolePrintf("\n"); konan::consolePrintf("Allocation histogram:\n"); @@ -227,18 +247,23 @@ public: keys[index++] = it.first; } std::sort(keys.begin(), keys.end()); - for (auto* it : keys) { + int perLine = 4; + int count = 0; + for (auto it : keys) { konan::consolePrintf( - "%d bytes -> %d times\n", it, (*allocationHistogram)[it]); + "%d bytes -> %d times ", it, (*allocationHistogram)[it]); + if (++count % perLine == (perLine - 1) || (count == keys.size())) + konan::consolePrintf("\n"); } + uint64_t allAddRefs = addRefs + atomicAddRefs; uint64_t allReleases = releaseRefs + atomicReleaseRefs + releaseCyclicRefs; - konan::consolePrintf("AddRefs:\t%lld/%lld (%lf%% of atomic)\n" - "Releases:\t%lld/%lld (%lf%% of atomic)\n" - "ReleaseRefs for cycle collector : %lld (%lf%% of cyclic)\n", + konan::consolePrintf("AddRefs:\t%lld/%lld (%.2lf%% of atomic)\n" + "Releases:\t%lld/%lld (%.2lf%% of atomic)\n" + "ReleaseRefs affecting cycle collector : %lld (%.2lf%% of cyclic)\n", addRefs, atomicAddRefs, percents(atomicAddRefs, allAddRefs), - releaseRefs, atomicReleaseRefs, percents(atomicAddRefs, allReleases), + releaseRefs, atomicReleaseRefs, percents(atomicReleaseRefs, allReleases), releaseCyclicRefs, percents(releaseCyclicRefs, allReleases)); } }; @@ -283,19 +308,17 @@ struct MemoryState { #if COLLECT_STATISTIC #define CONTAINER_ALLOC_STAT(state, size, container) state->statistic.incAlloc(size, container); - #define CONTAINER_FREE_STAT(state, container) #define CONTAINER_DESTROY_STAT(state, container) \ state->statistic.incFree(container); #define OBJECT_ALLOC_STAT(state, size, object) \ - state->statistic.incAlloc(size, object); - #define OBJECT_FREE_STAT(state, size, object) \ - state->statistic.incFree(object); - #define UPDATE_REF_STAT(state, oldRef, newRef, slot) \ - state->statistic.incUpdateRef(oldRef, newRef); - #define UPDATE_ADDREF_STAT(state, obj, atomic) \ - state->statistic.incAddRef(obj, atomic); - #define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic) \ - state->statistic.incReleaseRef(obj, atomic, cyclic); + state->statistic.incAlloc(size, object); \ + state->statistic.incAddRef(object->container(), 0, 0); + #define UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) \ + state->statistic.incUpdateRef(oldRef, newRef, stack); + #define UPDATE_ADDREF_STAT(state, obj, atomic, stack) \ + state->statistic.incAddRef(obj, atomic, stack); + #define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic, stack) \ + state->statistic.incReleaseRef(obj, atomic, cyclic, stack); #define INIT_STAT(state) \ state->statistic.init(); #define DEINIT_STAT(state) \ @@ -305,13 +328,11 @@ struct MemoryState { MemoryStatistic statistic; #else #define CONTAINER_ALLOC_STAT(state, size, container) - #define CONTAINER_FREE_STAT(state, container) #define CONTAINER_DESTROY_STAT(state, container) #define OBJECT_ALLOC_STAT(state, size, object) - #define OBJECT_FREE_STAT(state, object) - #define UPDATE_REF_STAT(state, oldRef, newRef, slot) - #define UPDATE_ADDREF_STAT(state, obj, atomic) - #define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic) + #define UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) + #define UPDATE_ADDREF_STAT(state, obj, atomic, stack) + #define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic, stack) #define INIT_STAT(state) #define DEINIT_STAT(state) #define PRINT_STAT(state) @@ -330,16 +351,12 @@ struct MemoryState { #endif #define CONTAINER_ALLOC_TRACE(state, size, container) \ MEMORY_LOG("Container alloc %d at %p\n", size, container) -#define CONTAINER_FREE_TRACE(state, container) \ - MEMORY_LOG("Container free %p\n", container) #define CONTAINER_DESTROY_TRACE(state, container) \ MEMORY_LOG("Container destroy %p\n", container) #define OBJECT_ALLOC_TRACE(state, size, object) \ MEMORY_LOG("Object alloc %d at %p\n", size, object) -#define OBJECT_FREE_TRACE(state, object) \ - MEMORY_LOG("Object free %p\n", object) -#define UPDATE_REF_TRACE(state, oldRef, newRef, slot) \ - MEMORY_LOG("UpdateRef *%p: %p -> %p\n", slot, oldRef, newRef) +#define UPDATE_REF_TRACE(state, oldRef, newRef, slot, stack) \ + MEMORY_LOG("UpdateRef %s*%p: %p -> %p\n", stack ? "stack " : "", slot, oldRef, newRef) // Events macro definitions. // Called on worker's memory init. @@ -353,10 +370,6 @@ struct MemoryState { #define CONTAINER_ALLOC_EVENT(state, size, container) \ CONTAINER_ALLOC_STAT(state, size, container) \ CONTAINER_ALLOC_TRACE(state, size, container) -// Called on container freeing (memory is still in use). -#define CONTAINER_FREE_EVENT(state, container) \ - CONTAINER_FREE_STAT(state, container) \ - CONTAINER_FREE_TRACE(state, container) // Called on container destroy (memory is released to allocator). #define CONTAINER_DESTROY_EVENT(state, container) \ CONTAINER_DESTROY_STAT(state, container) \ @@ -370,9 +383,9 @@ struct MemoryState { OBJECT_FREE_STAT(state, size, object) \ OBJECT_FREE_TRACE(state, object) // Reference in memory is being updated. -#define UPDATE_REF_EVENT(state, oldRef, newRef, slot) \ - UPDATE_REF_STAT(state, oldRef, newRef, slot) \ - UPDATE_REF_TRACE(state, oldRef, newRef, slot) +#define UPDATE_REF_EVENT(state, oldRef, newRef, slot, stack) \ + UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) \ + UPDATE_REF_TRACE(state, oldRef, newRef, slot, stack) // Infomation shall be printed as worker is exiting. #define PRINT_EVENT(state) \ PRINT_STAT(state) @@ -494,7 +507,7 @@ void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) { for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { ObjHeader** location = reinterpret_cast( reinterpret_cast(body) + typeInfo->objOffsets_[index]); - UpdateRef(location, nullptr); + ZeroHeapRef(location); } } @@ -574,9 +587,9 @@ inline void scheduleDestroyContainer(MemoryState* state, ContainerHeader* contai processFinalizerQueue(state); } #else - atomicAdd(&allocCount, -1); - CONTAINER_DESTROY_EVENT(state, container) konanFreeMemory(container); + atomicAdd(&allocCount, -1); + CONTAINER_DESTROY_EVENT(state, container); #endif } @@ -585,7 +598,6 @@ inline void scheduleDestroyContainer(MemoryState* state, ContainerHeader* contai template inline void IncrementRC(ContainerHeader* container) { container->incRefCount(); - UPDATE_ADDREF_STAT(memoryState, container, Atomic); } template @@ -593,7 +605,6 @@ inline void DecrementRC(ContainerHeader* container) { if (container->decRefCount() == 0) { FreeContainer(container); } - UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false); } #else // USE_GC @@ -606,13 +617,11 @@ template inline void IncrementRC(ContainerHeader* container) { container->incRefCount(); container->setColorUnlessGreen(CONTAINER_TAG_GC_BLACK); - UPDATE_ADDREF_STAT(memoryState, container, Atomic); } template inline void DecrementRC(ContainerHeader* container) { if (container->decRefCount() == 0) { - UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false); FreeContainer(container); } else if (UseCycleCollector) { // Possible root. RuntimeAssert(!Atomic, "Cycle collector shalln't be used with shared objects yet"); @@ -623,7 +632,6 @@ inline void DecrementRC(ContainerHeader* container) { // Also do not use cycle collector for provable acyclic objects. int color = container->color(); if (color != CONTAINER_TAG_GC_PURPLE && color != CONTAINER_TAG_GC_GREEN) { - UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, true); container->setColorAssertIfGreen(CONTAINER_TAG_GC_PURPLE); if (!container->buffered()) { container->setBuffered(); @@ -633,8 +641,6 @@ inline void DecrementRC(ContainerHeader* container) { GarbageCollect(); } } - } else { - UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false); } } } @@ -866,7 +872,7 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) { auto* childContainer = ref->container(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); if (Shareable(childContainer)) { - UpdateRef(location, nullptr); + ZeroHeapRef(location); } else { toVisit.push_front(childContainer); } @@ -877,23 +883,53 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) { } #endif -inline void AddRef(ContainerHeader* header) { +inline bool needAtomicAccess(ContainerHeader* container) { + return container->shareable(); +} + +inline bool canBeCyclic(ContainerHeader* container) { + if (container->refCount() == 1) return false; + if (container->color() == CONTAINER_TAG_GC_GREEN) return false; + return true; +} + +ALWAYS_INLINE inline void addRef(ContainerHeader* container) { // Looking at container type we may want to skip AddRef() totally // (non-escaping stack objects, constant objects). - switch (header->refCount_ & CONTAINER_TAG_MASK) { + switch (container->refCount_ & CONTAINER_TAG_MASK) { case CONTAINER_TAG_STACK: break; case CONTAINER_TAG_NORMAL: - IncrementRC(header); + IncrementRC(container); break; /* case CONTAINER_TAG_FROZEN: case CONTAINER_TAG_ATOMIC: */ default: - IncrementRC(header); + IncrementRC(container); break; } } -inline void ReleaseRef(ContainerHeader* header) { +ALWAYS_INLINE inline void AddHeapRef(ContainerHeader* container) { + UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 0); + addRef(container); +} + +ALWAYS_INLINE inline void AddHeapRef(ObjHeader* header) { + auto* container = header->container(); + if (container != nullptr) AddHeapRef(container); +} + +ALWAYS_INLINE inline void AddStackRef(ContainerHeader* container) { + UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 1); + addRef(container); +} + +ALWAYS_INLINE inline void AddStackRef(ObjHeader* header) { + auto* container = header->container(); + if (container != nullptr) AddStackRef(container); +} + +ALWAYS_INLINE inline void releaseRef(ContainerHeader* header) { // Looking at container type we may want to skip ReleaseRef() totally // (non-escaping stack objects, constant objects). switch (header->tag()) { @@ -909,6 +945,26 @@ inline void ReleaseRef(ContainerHeader* header) { } } +ALWAYS_INLINE inline void ReleaseHeapRef(ContainerHeader* container) { + UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 0); + releaseRef(container); +} + +ALWAYS_INLINE inline void ReleaseStackRef(ContainerHeader* container) { + UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 1); + releaseRef(container); +} + +ALWAYS_INLINE inline void ReleaseHeapRef(ObjHeader* header) { + auto* container = header->container(); + if (container != nullptr) ReleaseHeapRef(container); +} + +ALWAYS_INLINE inline void ReleaseStackRef(ObjHeader* header) { + auto* container = header->container(); + if (container != nullptr) ReleaseStackRef(container); +} + // We use first slot as place to store frame-local arena container. // TODO: create ArenaContainer object on the stack, so that we don't // do two allocations per frame (ArenaContainer + actual container). @@ -961,7 +1017,7 @@ void ObjHeader::destroyMetaObject(TypeInfo** location) { *const_cast(location) = meta->typeInfo_; if (meta->counter_ != nullptr) { WeakReferenceCounterClear(meta->counter_); - UpdateRef(&meta->counter_, nullptr); + ZeroHeapRef(&meta->counter_); } #ifdef KONAN_OBJC_INTEROP @@ -1029,8 +1085,6 @@ void FreeContainer(ContainerHeader* container) { RuntimeAssert(container != nullptr, "this kind of container shalln't be freed"); auto state = memoryState; - CONTAINER_FREE_EVENT(state, container) - if (isAggregatingFrozenContainer(container)) { FreeAggregatingFrozenContainer(container); return; @@ -1040,7 +1094,7 @@ void FreeContainer(ContainerHeader* container) { // Now let's clean all object's fields in this container. traverseContainerObjectFields(container, [](ObjHeader** location) { - UpdateRef(location, nullptr); + ZeroHeapRef(location); }); // And release underlying memory. @@ -1177,28 +1231,28 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun return result; } -inline void AddRef(const ObjHeader* object) { +inline void addRef(const ObjHeader* object) { auto* container = object->container(); if (container != nullptr) { MEMORY_LOG("AddRef on %p in %p\n", object, container) - AddRef(container); + addRef(container); } } -inline void ReleaseRef(const ObjHeader* object) { +inline void releaseRef(const ObjHeader* object) { auto* container = object->container(); if (container != nullptr) { MEMORY_LOG("ReleaseRef on %p in %p\n", object, container) - ReleaseRef(container); + releaseRef(container); } } void AddRefFromAssociatedObject(const ObjHeader* object) { - AddRef(object); + AddHeapRef(const_cast(object)); } void ReleaseRefFromAssociatedObject(const ObjHeader* object) { - ReleaseRef(object); + ReleaseHeapRef(const_cast(object)); } extern "C" { @@ -1304,8 +1358,7 @@ OBJ_GETTER(InitInstance, } ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); - MEMORY_LOG("Calling UpdateRef from InitInstance\n") - UpdateRef(location, object); + UpdateHeapRef(location, object); #if KONAN_NO_EXCEPTIONS ctor(object); return object; @@ -1314,8 +1367,8 @@ OBJ_GETTER(InitInstance, ctor(object); return object; } catch (...) { - UpdateRef(OBJ_RESULT, nullptr); - UpdateRef(location, nullptr); + ZeroStackRef(OBJ_RESULT); + ZeroHeapRef(location); throw; } #endif @@ -1330,7 +1383,7 @@ OBJ_GETTER(InitSharedInstance, RETURN_OBJ(value); } ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); - UpdateRef(location, object); + UpdateHeapRef(location, object); #if KONAN_NO_EXCEPTIONS ctor(object); FreezeSubgraph(object); @@ -1341,8 +1394,8 @@ OBJ_GETTER(InitSharedInstance, FreezeSubgraph(object); return object; } catch (...) { - UpdateRef(OBJ_RESULT, nullptr); - UpdateRef(location, nullptr); + ZeroStackRef(OBJ_RESULT); + ZeroHeapRef(location); throw; } #endif @@ -1360,37 +1413,57 @@ OBJ_GETTER(InitSharedInstance, } ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); RuntimeAssert(object->container()->normal() , "Shared object cannot be co-allocated"); - MEMORY_LOG("Calling UpdateRef from InitSharedInstance\n") - UpdateRef(localLocation, object); + UpdateHeapRef(localLocation, object); #if KONAN_NO_EXCEPTIONS ctor(object); FreezeSubgraph(object); - UpdateRef(location, object); - __sync_synchronize(); + UpdateHeapRef(location, object); + synchronize(); return object; #else try { ctor(object); FreezeSubgraph(object); - UpdateRef(location, object); - __sync_synchronize(); + UpdateHeapRef(location, object); + synchronize(); return object; } catch (...) { - UpdateRef(OBJ_RESULT, nullptr); - UpdateRef(location, nullptr); - UpdateRef(localLocation, nullptr); - __sync_synchronize(); + ZeroStackRef(OBJ_RESULT); + ZeroHeapRef(location); + ZeroHeapRef(localLocation); + synchronize(); throw; } #endif #endif } -void SetRef(ObjHeader** location, const ObjHeader* object) { - MEMORY_LOG("SetRef *%p: %p\n", location, object) +void SetHeapRef(ObjHeader** location, const ObjHeader* object) { + MEMORY_LOG("SetHeapRef *%p: %p\n", location, object) + UPDATE_REF_EVENT(memoryState, *location, object, location, 0); *const_cast(location) = object; if (object != nullptr) - AddRef(object); + AddHeapRef(const_cast(object)); +} + +void ZeroHeapRef(ObjHeader** location) { + MEMORY_LOG("ZeroHeapRef %p\n", location) + auto* value = *location; + if (value != nullptr) { + UPDATE_REF_EVENT(memoryState, value, nullptr, location, 0); + *location = nullptr; + ReleaseHeapRef(value); + } +} + +void ZeroStackRef(ObjHeader** location) { + MEMORY_LOG("ZeroStackRef %p\n", location) + auto* value = *location; + if (value != nullptr) { + UPDATE_REF_EVENT(memoryState, value, nullptr, location, 1); + *location = nullptr; + ReleaseStackRef(value); + } } ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { @@ -1406,60 +1479,71 @@ ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) { return reinterpret_cast(reinterpret_cast(&chunk->arena) | ARENA_BIT); } -void UpdateRef(ObjHeader** location, const ObjHeader* object) { +ALWAYS_INLINE void updateRef(ObjHeader** location, const ObjHeader* object) { RuntimeAssert(!isArenaSlot(location), "must not be a slot"); ObjHeader* old = *location; - UPDATE_REF_EVENT(memoryState, old, object, location) if (old != object) { if (object != nullptr) { - AddRef(object); + addRef(object); } *const_cast(location) = object; if (reinterpret_cast(old) > 1) { - ReleaseRef(old); + releaseRef(old); } } } -inline ObjHeader** slotAddressFor(ObjHeader** returnSlot, const ObjHeader* value) { +void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { + UPDATE_REF_EVENT(memoryState, *location, object, location, 1); + updateRef(location, object); +} + +void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { + UPDATE_REF_EVENT(memoryState, *location, object, location, 0); + updateRef(location, object); +} + +ALWAYS_INLINE inline ObjHeader** slotAddressFor(ObjHeader** returnSlot, const ObjHeader* value) { if (!isArenaSlot(returnSlot)) return returnSlot; // Not a subject of reference counting. if (value == nullptr || !isRefCounted(value)) return nullptr; return initedArena(asArenaSlot(returnSlot))->getSlot(); } -inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) { +ALWAYS_INLINE inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) { returnSlot = slotAddressFor(returnSlot, value); if (returnSlot == nullptr) return; ObjHeader* old = *returnSlot; + UPDATE_REF_EVENT(memoryState, old, value, returnSlot, 1); *const_cast(returnSlot) = value; if (old != nullptr) { - ReleaseRef(old); + ReleaseStackRef(old); } } void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* value) { returnSlot = slotAddressFor(returnSlot, value); if (returnSlot == nullptr) return; - UpdateRef(returnSlot, value); + UpdateStackRef(returnSlot, value); } -void UpdateRefIfNull(ObjHeader** location, const ObjHeader* object) { +void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { if (object != nullptr) { #if KONAN_NO_THREADS ObjHeader* old = *location; if (old == nullptr) { - AddRef(object); + AddHeapRef(const_cast(object)); *const_cast(location) = object; } #else - AddRef(object); + AddHeapRef(const_cast(object)); auto old = __sync_val_compare_and_swap(location, nullptr, const_cast(object)); if (old != nullptr) { // Failed to store, was not null. - ReleaseRef(object); + ReleaseHeapRef(const_cast(object)); } #endif + UPDATE_REF_EVENT(memoryState, old, object, location, 0); } } @@ -1467,9 +1551,25 @@ void EnterFrame(ObjHeader** start, int parameters, int count) { MEMORY_LOG("EnterFrame %p .. %p\n", start, start + count + parameters) } +inline void releaseStackRefs(MemoryState* state, ObjHeader** start, int count) { + MEMORY_LOG("ReleaseStackRefs %p .. %p\n", start, start + count) + ObjHeader** current = start; + while (count-- > 0) { + ObjHeader* object = *current; + if (object != nullptr) { + UPDATE_REF_EVENT(memoryState, object, nullptr, current, 1); + ReleaseStackRef(object); + // Just for sanity, optional. + *current = nullptr; + } + current++; + } +} + void LeaveFrame(ObjHeader** start, int parameters, int count) { MEMORY_LOG("LeaveFrame %p .. %p\n", start, start + count + parameters) - ReleaseRefs(start + parameters + kFrameOverlaySlots, count - kFrameOverlaySlots - parameters); + auto* state = memoryState; + releaseStackRefs(state, start + parameters + kFrameOverlaySlots, count - kFrameOverlaySlots - parameters); if (*start != nullptr) { auto arena = initedArena(start); MEMORY_LOG("LeaveFrame: free arena %p\n", arena) @@ -1479,20 +1579,6 @@ void LeaveFrame(ObjHeader** start, int parameters, int count) { } } -void ReleaseRefs(ObjHeader** start, int count) { - MEMORY_LOG("ReleaseRefs %p .. %p\n", start, start + count) - ObjHeader** current = start; - auto state = memoryState; - while (count-- > 0) { - ObjHeader* object = *current; - if (object != nullptr) { - ReleaseRef(object); - // Just for sanity, optional. - *current = nullptr; - } - current++; - } -} #if USE_GC @@ -1599,14 +1685,14 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) { KNativePtr CreateStablePointer(KRef any) { if (any == nullptr) return nullptr; - AddRef(any); + AddHeapRef(any); return reinterpret_cast(any); } void DisposeStablePointer(KNativePtr pointer) { if (pointer == nullptr) return; KRef ref = reinterpret_cast(pointer); - ReleaseRef(ref); + ReleaseHeapRef(ref); } OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { @@ -1615,11 +1701,10 @@ OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { } OBJ_GETTER(AdoptStablePointer, KNativePtr pointer) { -#ifndef KONAN_NO_THREADS - __sync_synchronize(); -#endif + synchronize(); KRef ref = reinterpret_cast(pointer); - UpdateRef(OBJ_RESULT, nullptr); + // TODO: as it is called from C runtime, + ZeroHeapRef(OBJ_RESULT); // Somewhat hacky. *OBJ_RESULT = ref; return ref; @@ -1909,17 +1994,17 @@ void MutationCheck(ObjHeader* obj) { if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj); } -OBJ_GETTER(SwapRefLocked, +OBJ_GETTER(SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) { lock(spinlock); ObjHeader* oldValue = *location; // We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock. if (oldValue == expectedValue) { - SetRef(location, newValue); + SetHeapRef(location, newValue); } else { // We create an additional reference to the [oldValue] in the return slot. if (oldValue != nullptr && isRefCounted(oldValue)) { - AddRef(oldValue); + AddHeapRef(oldValue); } } unlock(spinlock); @@ -1929,14 +2014,14 @@ OBJ_GETTER(SwapRefLocked, return oldValue; } -void SetRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) { +void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) { lock(spinlock); ObjHeader* oldValue = *location; // We do not use UpdateRef() here to avoid having ReleaseRef() on old value under the lock. - SetRef(location, newValue); + SetHeapRef(location, newValue); unlock(spinlock); if (oldValue != nullptr) - ReleaseRef(oldValue); + ReleaseHeapRef(oldValue); } OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) { @@ -1944,7 +2029,7 @@ OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) { ObjHeader* value = *location; // We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock. if (value != nullptr) - AddRef(value); + AddHeapRef(value); unlock(spinlock); updateReturnRefAdded(OBJ_RESULT, value); return value; @@ -1995,7 +2080,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what } if (!acyclic) return false; } - UpdateRef(reinterpret_cast( + UpdateHeapRef(reinterpret_cast( reinterpret_cast(where) + where->type_info()->objOffsets_[index]), what); // Fence on updated location? return true; diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index c876d32261c..dc093b924d8 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -492,7 +492,7 @@ void WeakReferenceCounterClear(ObjHeader* counter); // Reference management scheme we use assumes significant degree of flexibility, so that // one could implement either pure reference counting scheme, or tracing collector without // much ado. -// Most important primitive is UpdateRef() API, which modifies location to use new +// Most important primitive is Update*Ref() API, which modifies location to use new // object reference. In pure reference counted scheme it will check old value, // decrement reference, increment counter on the new value, and store it into the field. // In tracing collector-like scheme, only field updates counts, and all other operations are @@ -508,23 +508,27 @@ void WeakReferenceCounterClear(ObjHeader* counter); // in intermediate frames when throwing // -// Sets location. -void SetRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; -// Updates location. -void UpdateRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Sets heap location. +void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Zeroes heap location. +void ZeroHeapRef(ObjHeader** location) RUNTIME_NOTHROW; +// Zeroes stack location. +void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW; +// Updates stack location. +void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Updates heap/static data location. +void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates location if it is null, atomically. -void UpdateRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates reference in return slot. void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW; // Compares and swaps reference with taken lock. -OBJ_GETTER(SwapRefLocked, +OBJ_GETTER(SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW; // Sets reference with taken lock. -void SetRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW; +void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW; // Reads reference with taken lock. OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW; -// Optimization: release all references in range. -void ReleaseRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; // Called on frame enter, if it has object slots. void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; // Called on frame leave, if it has object slots. @@ -563,16 +567,16 @@ class ObjHolder { ObjHolder() : obj_(nullptr) {} explicit ObjHolder(const ObjHeader* obj) { - ::SetRef(&obj_, obj); + ::SetHeapRef(&obj_, obj); } ~ObjHolder() { - ::UpdateRef(&obj_, nullptr); + ::ZeroHeapRef(&obj_); } ObjHeader* obj() { return obj_; } const ObjHeader* obj() const { return obj_; } ObjHeader** slot() { return &obj_; } - void clear() { ::UpdateRef(&obj_, nullptr); } + void clear() { ::ZeroHeapRef(&obj_); } private: ObjHeader* obj_; @@ -586,7 +590,7 @@ class KRefSharedHolder { } inline void init(ObjHeader* obj) { - SetRef(slotToInit(), obj); + SetHeapRef(slotToInit(), obj); } inline ObjHeader* ref() const { @@ -596,7 +600,7 @@ class KRefSharedHolder { inline void dispose() { verifyRefOwner(); - UpdateRef(&obj_, nullptr); + ZeroHeapRef(&obj_); } private: diff --git a/runtime/src/main/cpp/Weak.cpp b/runtime/src/main/cpp/Weak.cpp index df2f0e490f5..d6fca1ae139 100644 --- a/runtime/src/main/cpp/Weak.cpp +++ b/runtime/src/main/cpp/Weak.cpp @@ -65,7 +65,7 @@ OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) { ObjHolder counterHolder; // Cast unneeded, just to emphasize we store an object reference as void*. ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast(referred), counterHolder.slot()); - UpdateRefIfNull(&meta->counter_, counter); + UpdateHeapRefIfNull(&meta->counter_, counter); } RETURN_OBJ(meta->counter_); } diff --git a/runtime/src/main/cpp/Worker.cpp b/runtime/src/main/cpp/Worker.cpp index 3f6016f42ea..5d5ccf3cbdd 100644 --- a/runtime/src/main/cpp/Worker.cpp +++ b/runtime/src/main/cpp/Worker.cpp @@ -67,9 +67,8 @@ KNativePtr transfer(KRef object, KInt mode) { case UNCHECKED: if (!ClearSubgraphReferences(object, mode == CHECKED)) { // Release reference to the object, as it is not being managed by ObjHolder. - UpdateRef(&object, nullptr); + ZeroHeapRef(&object); ThrowWorkerInvalidState(); - return nullptr; } return object; } diff --git a/runtime/src/main/kotlin/kotlin/native/internal/test/Launcher.kt b/runtime/src/main/kotlin/kotlin/native/internal/test/Launcher.kt index e2ee278c144..1b550be5d66 100644 --- a/runtime/src/main/kotlin/kotlin/native/internal/test/Launcher.kt +++ b/runtime/src/main/kotlin/kotlin/native/internal/test/Launcher.kt @@ -31,3 +31,7 @@ fun worker(args: Array) { worker.requestTermination().result exitProcess(result) } + +fun mainNoExit(args: Array) { + testLauncherEntryPoint(args) +} \ No newline at end of file