azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Distinguish stack/heap writes and collect stats. (#2794)

Browse Source
This commit is contained in:
Nikolay Igotti
2019-03-22 15:50:42 +03:00
committed by GitHub
parent 3a861b7917
commit c3b2b47403
19 changed files with 325 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
+1
View File
@@ -181,6 +181,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
when {
arguments.generateWorkerTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.WORKER)
arguments.generateTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.MAIN_THREAD)
arguments.generateNoExitTestRunner -> put(GENERATE_TEST_RUNNER, TestRunnerKind.MAIN_THREAD_NO_EXIT)
else -> put(GENERATE_TEST_RUNNER, TestRunnerKind.NONE)
}
// We need to download dependencies only if we use them ( = there are files to compile).
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt
+3
View File
@@ -29,6 +29,9 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
@Argument(value = "-generate-worker-test-runner",
shortName = "-trw", description = "Produce a worker runner for unit tests")
var generateWorkerTestRunner = false
@Argument(value = "-generate-no-exit-test-runner",
shortName = "-trn", description = "Produce a runner for unit tests not forcing exit")
var generateNoExitTestRunner = false
@Argument(value="-include-binary", deprecatedName = "-includeBinary", shortName = "-ib", valueDescription = "<path>", description = "Pack external binary within the klib")
var includeBinaries: Array<String>? = null
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt
+3 -3
View File
@@ -849,7 +849,7 @@ internal class CAdapterGenerator(
|#define RUNTIME_USED __attribute__((used))
|
|extern "C" {
|void UpdateRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
|void UpdateHeapRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
|KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW;
|KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW;
|void* CreateStablePointer(KObjHeader*) RUNTIME_NOTHROW;
@@ -866,10 +866,10 @@ internal class CAdapterGenerator(
|public:
| KObjHolder() : obj_(nullptr) {}
| explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) {
| UpdateRef(&obj_, obj);
| UpdateHeapRef(&obj_, obj);
| }
| ~KObjHolder() {
| UpdateRef(&obj_, nullptr);
| UpdateHeapRef(&obj_, nullptr);
| }
| KObjHeader* obj() { return obj_; }
| KObjHeader** slot() { return &obj_; }
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/TestRunnerKind.kt
+2 -1
View File
@@ -9,5 +9,6 @@ package org.jetbrains.kotlin.backend.konan
enum class TestRunnerKind {
NONE,
MAIN_THREAD,
WORKER
WORKER,
MAIN_THREAD_NO_EXIT
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt
+13 -5
View File
@@ -270,7 +270,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
val value = LLVMBuildLoad(builder, address, name)!!
if (isObjectRef(value) && isVar) {
val slot = alloca(LLVMTypeOf(value), variableLocation = null)
storeAny(value, slot)
storeStackRef(value, slot)
}
return value
}
@@ -281,9 +281,17 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
LLVMBuildStore(builder, value, ptr)
}
fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef) {
fun storeHeapRef(value: LLVMValueRef, ptr: LLVMValueRef) {
updateRef(value, ptr, onStack = false)
}
fun storeStackRef(value: LLVMValueRef, ptr: LLVMValueRef) {
updateRef(value, ptr, onStack = true)
}
fun storeAny(value: LLVMValueRef, ptr: LLVMValueRef, onStack: Boolean) {
if (isObjectRef(value)) {
updateRef(value, ptr)
if (onStack) storeStackRef(value, ptr) else storeHeapRef(value, ptr)
} else {
LLVMBuildStore(builder, value, ptr)
}
@@ -302,8 +310,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
call(context.llvm.updateReturnRefFunction, listOf(address, value))
}
private fun updateRef(value: LLVMValueRef, address: LLVMValueRef) {
call(context.llvm.updateRefFunction, listOf(address, value))
private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, onStack: Boolean) {
call(if (onStack) context.llvm.updateStackRefFunction else context.llvm.updateHeapRefFunction, listOf(address, value))
}
//-------------------------------------------------------------------------//
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt
+2 -1
View File
@@ -401,7 +401,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
val initInstanceFunction = importRtFunction("InitInstance")
val initSharedInstanceFunction = importRtFunction("InitSharedInstance")
val updateReturnRefFunction = importRtFunction("UpdateReturnRef")
val updateRefFunction = importRtFunction("UpdateRef")
val updateHeapRefFunction = importRtFunction("UpdateHeapRef")
val updateStackRefFunction = importRtFunction("UpdateStackRef")
val enterFrameFunction = importRtFunction("EnterFrame")
val leaveFrameFunction = importRtFunction("LeaveFrame")
val getReturnSlotIfArenaFunction = importRtFunction("GetReturnSlotIfArena")
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EntryPoint.kt
+1
View File
@@ -51,6 +51,7 @@ private fun defaultEntryName(config: CompilerConfiguration): String =
when (config.get(KonanConfigKeys.GENERATE_TEST_RUNNER)) {
TestRunnerKind.MAIN_THREAD -> "kotlin.native.internal.test.main"
TestRunnerKind.WORKER -> "kotlin.native.internal.test.worker"
TestRunnerKind.MAIN_THREAD_NO_EXIT -> "kotlin.native.internal.test.mainNoExit"
else -> "main"
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
+8 -8
View File
@@ -371,7 +371,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
val address = context.llvmDeclarations.forStaticField(it).storage
if (it.storageClass == FieldStorage.SHARED)
freeze(initialization, currentCodeContext.exceptionHandler)
storeAny(initialization, address)
storeAny(initialization, address, false)
}
}
}
@@ -385,7 +385,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
if (it.storageClass == FieldStorage.THREAD_LOCAL) {
val initialization = evaluateExpression(it.initializer!!.expression)
val address = context.llvmDeclarations.forStaticField(it).storage
storeAny(initialization, address)
storeAny(initialization, address, false)
}
}
}
@@ -397,10 +397,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
// Only if a subject for memory management.
if (it.type.binaryTypeIsReference() && it.storageClass == FieldStorage.THREAD_LOCAL) {
val address = context.llvmDeclarations.forStaticField(it).storage
storeAny(codegen.kNullObjHeaderPtr, address)
storeHeapRef(codegen.kNullObjHeaderPtr, address)
}
}
context.llvm.objects.forEach { storeAny(codegen.kNullObjHeaderPtr, it) }
context.llvm.objects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) }
ret(null)
}
@@ -410,10 +410,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
.forEach {
if (it.type.binaryTypeIsReference() && it.storageClass != FieldStorage.THREAD_LOCAL) {
val address = context.llvmDeclarations.forStaticField(it).storage
storeAny(codegen.kNullObjHeaderPtr, address)
storeHeapRef(codegen.kNullObjHeaderPtr, address)
}
}
context.llvm.sharedObjects.forEach { storeAny(codegen.kNullObjHeaderPtr, it) }
context.llvm.sharedObjects.forEach { storeHeapRef(codegen.kNullObjHeaderPtr, it) }
ret(null)
}
}
@@ -1469,7 +1469,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
listOf(functionGenerationContext.bitcast(codegen.kObjHeaderPtr, thisPtr)),
Lifetime.IRRELEVANT, ExceptionHandler.Caller)
}
functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner))
functionGenerationContext.storeAny(valueToAssign, fieldPtrOfClass(thisPtr, value.symbol.owner), false)
} else {
assert(value.receiver == null)
val globalValue = context.llvmDeclarations.forStaticField(value.symbol.owner).storage
@@ -1477,7 +1477,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
functionGenerationContext.checkMainThread(currentCodeContext.exceptionHandler)
if (value.symbol.owner.storageClass == FieldStorage.SHARED)
functionGenerationContext.freeze(valueToAssign, currentCodeContext.exceptionHandler)
functionGenerationContext.storeAny(valueToAssign, globalValue)
functionGenerationContext.storeAny(valueToAssign, globalValue, false)
}
assert (value.type.isUnit())
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt
+3 -3
View File
@@ -24,7 +24,7 @@ internal class VariableManager(val functionGenerationContext: FunctionGeneration
inner class SlotRecord(val address: LLVMValueRef, val refSlot: Boolean, val isVar: Boolean) : Record {
override fun load() : LLVMValueRef = functionGenerationContext.loadSlot(address, isVar)
override fun store(value: LLVMValueRef) = functionGenerationContext.storeAny(value, address)
override fun store(value: LLVMValueRef) = functionGenerationContext.storeAny(value, address, true)
override fun address() : LLVMValueRef = this.address
override fun toString() = (if (refSlot) "refslot" else "slot") + " for ${address}"
}
@@ -72,7 +72,7 @@ internal class VariableManager(val functionGenerationContext: FunctionGeneration
val type = functionGenerationContext.getLLVMType(valueDeclaration.type)
val slot = functionGenerationContext.alloca(type, valueDeclaration.name.asString(), variableLocation)
if (value != null)
functionGenerationContext.storeAny(value, slot)
functionGenerationContext.storeAny(value, slot, true)
variables.add(SlotRecord(slot, functionGenerationContext.isObjectType(type), isVar))
contextVariablesToIndex[valueDeclaration] = index
return index
@@ -103,7 +103,7 @@ internal class VariableManager(val functionGenerationContext: FunctionGeneration
val index = variables.size
val slot = functionGenerationContext.alloca(type, variableLocation = null)
if (value != null)
functionGenerationContext.storeAny(value, slot)
functionGenerationContext.storeAny(value, slot, true)
variables.add(SlotRecord(slot, functionGenerationContext.isObjectType(type), true))
return index
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/objcexport/BlockPointerSupport.kt
+6 -4
View File
@@ -63,7 +63,7 @@ internal class BlockAdapterToFunctionGenerator(val objCExportCodeGenerator: ObjC
) {
val blockPtr = bitcast(pointerType(blockLiteralType), param(0))
val slot = structGep(blockPtr, 1)
storeAny(kNullObjHeaderPtr, slot) // TODO: can dispose_helper write to the block?
storeHeapRef(kNullObjHeaderPtr, slot) // TODO: can dispose_helper write to the block?
ret(null)
}.also {
@@ -84,7 +84,7 @@ internal class BlockAdapterToFunctionGenerator(val objCExportCodeGenerator: ObjC
// Kotlin reference was `memcpy`ed from src to dst, "revert" this:
storeRefUnsafe(kNullObjHeaderPtr, dstSlot)
// and copy properly:
storeAny(loadSlot(srcSlot, isVar = false), dstSlot)
storeHeapRef(loadSlot(srcSlot, isVar = false), dstSlot)
ret(null)
}.also {
@@ -126,7 +126,7 @@ internal class BlockAdapterToFunctionGenerator(val objCExportCodeGenerator: ObjC
assert(value.type == kObjHeaderPtr)
assert(slot.type == kObjHeaderPtrPtr)
storeAny(
store(
bitcast(int8TypePtr, value),
bitcast(pointerType(int8TypePtr), slot)
)
@@ -191,7 +191,9 @@ internal class BlockAdapterToFunctionGenerator(val objCExportCodeGenerator: ObjC
val slot = structGep(blockOnStack, 1)
listOf(bitcast(int8TypePtr, isa), flags, reserved, invoke, descriptor).forEachIndexed { index, value ->
storeAny(value, structGep(blockOnStackBase, index))
// Although value is actually on the stack, it's not in normal slot area, so we cannot handle it
// as if it was on the stack.
store(value, structGep(blockOnStackBase, index))
}
// Note: it is the slot in the block located on stack, so no need to manage it properly:
runtime/src/main/cpp/Arrays.cpp
+4 -4
View File
@@ -92,7 +92,7 @@ void Kotlin_Array_set(KRef thiz, KInt index, KConstRef value) {
ThrowArrayIndexOutOfBoundsException();
}
mutabilityCheck(thiz);
UpdateRef(ArrayAddressOfElementAt(array, index), value);
UpdateHeapRef(ArrayAddressOfElementAt(array, index), value);
}
KInt Kotlin_Array_getArrayLength(KConstRef thiz) {
@@ -107,7 +107,7 @@ void Kotlin_Array_fillImpl(KRef thiz, KInt fromIndex, KInt toIndex, KRef value)
}
mutabilityCheck(thiz);
for (KInt index = fromIndex; index < toIndex; ++index) {
UpdateRef(ArrayAddressOfElementAt(array, index), value);
UpdateHeapRef(ArrayAddressOfElementAt(array, index), value);
}
}
@@ -123,12 +123,12 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex,
mutabilityCheck(destination);
if (fromIndex >= toIndex) {
for (int index = 0; index < count; index++) {
UpdateRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
*ArrayAddressOfElementAt(array, fromIndex + index));
}
} else {
for (int index = count - 1; index >= 0; index--) {
UpdateRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
UpdateHeapRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
*ArrayAddressOfElementAt(array, fromIndex + index));
}
}
runtime/src/main/cpp/Atomic.cpp
+3 -3
View File
@@ -184,7 +184,7 @@ OBJ_GETTER(Kotlin_AtomicReference_compareAndSwap, KRef thiz, KRef expectedValue,
Kotlin_AtomicReference_checkIfFrozen(newValue);
// See Kotlin_AtomicReference_get() for explanations, why locking is needed.
AtomicReferenceLayout* ref = asAtomicReference(thiz);
RETURN_RESULT_OF(SwapRefLocked, &ref->value_, expectedValue, newValue, &ref->lock_);
RETURN_RESULT_OF(SwapHeapRefLocked, &ref->value_, expectedValue, newValue, &ref->lock_);
}
KBoolean Kotlin_AtomicReference_compareAndSet(KRef thiz, KRef expectedValue, KRef newValue) {
@@ -192,14 +192,14 @@ KBoolean Kotlin_AtomicReference_compareAndSet(KRef thiz, KRef expectedValue, KRe
// See Kotlin_AtomicReference_get() for explanations, why locking is needed.
AtomicReferenceLayout* ref = asAtomicReference(thiz);
ObjHolder holder;
auto old = SwapRefLocked(&ref->value_, expectedValue, newValue, &ref->lock_, holder.slot());
auto old = SwapHeapRefLocked(&ref->value_, expectedValue, newValue, &ref->lock_, holder.slot());
return old == expectedValue;
}
void Kotlin_AtomicReference_set(KRef thiz, KRef newValue) {
Kotlin_AtomicReference_checkIfFrozen(newValue);
AtomicReferenceLayout* ref = asAtomicReference(thiz);
SetRefLocked(&ref->value_, newValue, &ref->lock_);
SetHeapRefLocked(&ref->value_, newValue, &ref->lock_);
}
OBJ_GETTER(Kotlin_AtomicReference_get, KRef thiz) {
runtime/src/main/cpp/Atomic.h
+6
View File
@@ -60,4 +60,10 @@ ALWAYS_INLINE inline T atomicGet(volatile T* where) {
#endif
}
static ALWAYS_INLINE inline void synchronize() {
#ifndef KONAN_NO_THREADS
__sync_synchronize();
#endif
}
#endif // RUNTIME_ATOMIC_H
runtime/src/main/cpp/Exceptions.cpp
+2 -2
View File
@@ -215,13 +215,13 @@ void ThrowException(KRef exception) {
}
OBJ_GETTER(Kotlin_setUnhandledExceptionHook, KRef hook) {
RETURN_RESULT_OF(SwapRefLocked,
RETURN_RESULT_OF(SwapHeapRefLocked,
&currentUnhandledExceptionHook, currentUnhandledExceptionHook, hook, &currentUnhandledExceptionHookLock);
}
void OnUnhandledException(KRef throwable) {
ObjHolder handlerHolder;
auto* handler = SwapRefLocked(&currentUnhandledExceptionHook, currentUnhandledExceptionHook, nullptr,
auto* handler = SwapHeapRefLocked(&currentUnhandledExceptionHook, currentUnhandledExceptionHook, nullptr,
&currentUnhandledExceptionHookLock, handlerHolder.slot());
if (handler == nullptr) {
ReportUnhandledException(throwable);
runtime/src/main/cpp/Memory.cpp
+243 -158
View File
@@ -92,8 +92,8 @@ struct FrameOverlay {
FrameOverlay exportFrameOverlay;
// Current number of allocated containers.
int allocCount = 0;
int aliveMemoryStatesCount = 0;
volatile int allocCount = 0;
volatile int aliveMemoryStatesCount = 0;
// Forward declarations.
void FreeContainer(ContainerHeader* header);
@@ -102,11 +102,11 @@ void FreeContainer(ContainerHeader* header);
class MemoryStatistic {
public:
// UpdateRef per-object type counters.
uint64_t updateCounters[5][5];
uint64_t updateCounters[10][10];
// Alloc per container type counters.
uint64_t containerAllocs[5][2];
uint64_t containerAllocs[2];
// Free per container type counters.
uint64_t objectAllocs[5][2];
uint64_t objectAllocs[5];
// Histogram of allocation size distribution.
KStdUnorderedMap<int, int>* allocationHistogram;
// Number of allocation cache hits.
@@ -142,11 +142,11 @@ public:
allocationHistogram = nullptr;
}
void incAddRef(const ContainerHeader* header, bool atomic) {
void incAddRef(const ContainerHeader* header, bool atomic, int stack) {
if (atomic) atomicAddRefs++; else addRefs++;
}
void incReleaseRef(const ContainerHeader* header, bool atomic, bool cyclic) {
void incReleaseRef(const ContainerHeader* header, bool atomic, bool cyclic, int stack) {
if (atomic) {
RuntimeAssert(!cyclic, "Atomic updates cannot be cyclic yet");
atomicReleaseRefs++;
@@ -155,69 +155,89 @@ public:
}
}
void incUpdateRef(const ObjHeader* objOld, const ObjHeader* objNew) {
updateCounters[toIndex(objOld)][toIndex(objNew)]++;
void incUpdateRef(const ObjHeader* objOld, const ObjHeader* objNew, int stack) {
updateCounters[toIndex(objOld, stack)][toIndex(objNew, stack)]++;
}
void incAlloc(size_t size, const ContainerHeader* header) {
containerAllocs[toIndex(header)][0]++;
containerAllocs[0]++;
++(*allocationHistogram)[size];
}
void incFree(const ContainerHeader* header) {
containerAllocs[toIndex(header)][1]++;
containerAllocs[1]++;
}
void incAlloc(size_t size, const ObjHeader* header) {
objectAllocs[toIndex(header)][0]++;
objectAllocs[toIndex(header, 0)]++;
}
void incFree(const ObjHeader* header) {
objectAllocs[toIndex(header)][1]++;
}
static int toIndex(const ObjHeader* obj) {
static int toIndex(const ObjHeader* obj, int stack) {
if (reinterpret_cast<uintptr_t>(obj) > 1)
return toIndex(obj->container());
return toIndex(obj->container(), stack);
else
return 4;
return 4 + stack * 5;
}
static int toIndex(const ContainerHeader* header) {
if (header == nullptr) return 2; // permanent.
static int toIndex(const ContainerHeader* header, int stack) {
if (header == nullptr) return 2 + stack * 5; // permanent.
switch (header->tag()) {
case CONTAINER_TAG_NORMAL : return 0;
case CONTAINER_TAG_STACK : return 1;
case CONTAINER_TAG_FROZEN: return 3;
case CONTAINER_TAG_NORMAL : return 0 + stack * 5;
case CONTAINER_TAG_STACK : return 1 + stack * 5;
case CONTAINER_TAG_FROZEN: return 3 + stack * 5;
}
RuntimeAssert(false, "unknown container type");
return -1;
}
static double percents(uint64_t value, uint64_t all) {
return ((double)value / (double)all) * 100.0;
return all == 0 ? 0 : ((double)value / (double)all) * 100.0;
}
void printStatistic() {
konan::consolePrintf("\nMemory manager statistic:\n\n");
for (int i = 0; i < 2; i++) {
konan::consolePrintf("Container %s alloc: %lld, free: %lld\n",
indexToName[i], containerAllocs[i][0],
containerAllocs[i][1]);
konan::consolePrintf("Container alloc: %lld, free: %lld\n",
containerAllocs[0], containerAllocs[1]);
for (int i = 0; i < 5; i++) {
// Only normal and frozen can be allocated.
if (i == 0 || i == 3)
konan::consolePrintf("Object %s alloc: %lld\n", indexToName[i], objectAllocs[i]);
}
for (int i = 0; i < 2; i++) {
konan::consolePrintf("Object %s alloc: %lld, free: %lld\n",
indexToName[i], objectAllocs[i][0],
objectAllocs[i][1]);
}
konan::consolePrintf("\n");
uint64_t allUpdateRefs = 0, heapUpdateRefs = 0, stackUpdateRefs = 0;
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
allUpdateRefs += updateCounters[i][j];
if (i < 5 && j < 5)
heapUpdateRefs += updateCounters[i][j];
if (i >= 5 && j >= 5)
stackUpdateRefs += updateCounters[i][j];
}
}
konan::consolePrintf("Total updates: %lld, stack: %lld(%.2lf%%), heap: %lld(%.2lf%%)\n",
allUpdateRefs,
stackUpdateRefs, percents(stackUpdateRefs, allUpdateRefs),
heapUpdateRefs, percents(heapUpdateRefs, allUpdateRefs));
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 5; j++) {
konan::consolePrintf("UpdateRef[%s -> %s]: %lld\n",
indexToName[i], indexToName[j], updateCounters[i][j]);
if (updateCounters[i][j] != 0)
konan::consolePrintf("UpdateHeapRef[%s -> %s]: %lld (%.2lf%% of all, %.2lf%% of heap)\n",
indexToName[i], indexToName[j], updateCounters[i][j],
percents(updateCounters[i][j], allUpdateRefs),
percents(updateCounters[i][j], heapUpdateRefs));
}
}
for (int i = 5; i < 10; i++) {
for (int j = 5; j < 10; j++) {
if (updateCounters[i][j] != 0)
konan::consolePrintf("UpdateStackRef[%s -> %s]: %lld (%.2lf%% of all, %.2lf%% of stack)\n",
indexToName[i - 5], indexToName[j - 5],
updateCounters[i][j],
percents(updateCounters[i][j], allUpdateRefs),
percents(updateCounters[i][j], stackUpdateRefs));
}
}
konan::consolePrintf("\n");
konan::consolePrintf("Allocation histogram:\n");
@@ -227,18 +247,23 @@ public:
keys[index++] = it.first;
}
std::sort(keys.begin(), keys.end());
for (auto* it : keys) {
int perLine = 4;
int count = 0;
for (auto it : keys) {
konan::consolePrintf(
"%d bytes -> %d times\n", it, (*allocationHistogram)[it]);
"%d bytes -> %d times ", it, (*allocationHistogram)[it]);
if (++count % perLine == (perLine - 1) || (count == keys.size()))
konan::consolePrintf("\n");
}
uint64_t allAddRefs = addRefs + atomicAddRefs;
uint64_t allReleases = releaseRefs + atomicReleaseRefs + releaseCyclicRefs;
konan::consolePrintf("AddRefs:\t%lld/%lld (%lf%% of atomic)\n"
"Releases:\t%lld/%lld (%lf%% of atomic)\n"
"ReleaseRefs for cycle collector : %lld (%lf%% of cyclic)\n",
konan::consolePrintf("AddRefs:\t%lld/%lld (%.2lf%% of atomic)\n"
"Releases:\t%lld/%lld (%.2lf%% of atomic)\n"
"ReleaseRefs affecting cycle collector : %lld (%.2lf%% of cyclic)\n",
addRefs, atomicAddRefs, percents(atomicAddRefs, allAddRefs),
releaseRefs, atomicReleaseRefs, percents(atomicAddRefs, allReleases),
releaseRefs, atomicReleaseRefs, percents(atomicReleaseRefs, allReleases),
releaseCyclicRefs, percents(releaseCyclicRefs, allReleases));
}
};
@@ -283,19 +308,17 @@ struct MemoryState {
#if COLLECT_STATISTIC
#define CONTAINER_ALLOC_STAT(state, size, container) state->statistic.incAlloc(size, container);
#define CONTAINER_FREE_STAT(state, container)
#define CONTAINER_DESTROY_STAT(state, container) \
state->statistic.incFree(container);
#define OBJECT_ALLOC_STAT(state, size, object) \
state->statistic.incAlloc(size, object);
#define OBJECT_FREE_STAT(state, size, object) \
state->statistic.incFree(object);
#define UPDATE_REF_STAT(state, oldRef, newRef, slot) \
state->statistic.incUpdateRef(oldRef, newRef);
#define UPDATE_ADDREF_STAT(state, obj, atomic) \
state->statistic.incAddRef(obj, atomic);
#define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic) \
state->statistic.incReleaseRef(obj, atomic, cyclic);
state->statistic.incAlloc(size, object); \
state->statistic.incAddRef(object->container(), 0, 0);
#define UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) \
state->statistic.incUpdateRef(oldRef, newRef, stack);
#define UPDATE_ADDREF_STAT(state, obj, atomic, stack) \
state->statistic.incAddRef(obj, atomic, stack);
#define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic, stack) \
state->statistic.incReleaseRef(obj, atomic, cyclic, stack);
#define INIT_STAT(state) \
state->statistic.init();
#define DEINIT_STAT(state) \
@@ -305,13 +328,11 @@ struct MemoryState {
MemoryStatistic statistic;
#else
#define CONTAINER_ALLOC_STAT(state, size, container)
#define CONTAINER_FREE_STAT(state, container)
#define CONTAINER_DESTROY_STAT(state, container)
#define OBJECT_ALLOC_STAT(state, size, object)
#define OBJECT_FREE_STAT(state, object)
#define UPDATE_REF_STAT(state, oldRef, newRef, slot)
#define UPDATE_ADDREF_STAT(state, obj, atomic)
#define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic)
#define UPDATE_REF_STAT(state, oldRef, newRef, slot, stack)
#define UPDATE_ADDREF_STAT(state, obj, atomic, stack)
#define UPDATE_RELEASEREF_STAT(state, obj, atomic, cyclic, stack)
#define INIT_STAT(state)
#define DEINIT_STAT(state)
#define PRINT_STAT(state)
@@ -330,16 +351,12 @@ struct MemoryState {
#endif
#define CONTAINER_ALLOC_TRACE(state, size, container) \
MEMORY_LOG("Container alloc %d at %p\n", size, container)
#define CONTAINER_FREE_TRACE(state, container) \
MEMORY_LOG("Container free %p\n", container)
#define CONTAINER_DESTROY_TRACE(state, container) \
MEMORY_LOG("Container destroy %p\n", container)
#define OBJECT_ALLOC_TRACE(state, size, object) \
MEMORY_LOG("Object alloc %d at %p\n", size, object)
#define OBJECT_FREE_TRACE(state, object) \
MEMORY_LOG("Object free %p\n", object)
#define UPDATE_REF_TRACE(state, oldRef, newRef, slot) \
MEMORY_LOG("UpdateRef *%p: %p -> %p\n", slot, oldRef, newRef)
#define UPDATE_REF_TRACE(state, oldRef, newRef, slot, stack) \
MEMORY_LOG("UpdateRef %s*%p: %p -> %p\n", stack ? "stack " : "", slot, oldRef, newRef)
// Events macro definitions.
// Called on worker's memory init.
@@ -353,10 +370,6 @@ struct MemoryState {
#define CONTAINER_ALLOC_EVENT(state, size, container) \
CONTAINER_ALLOC_STAT(state, size, container) \
CONTAINER_ALLOC_TRACE(state, size, container)
// Called on container freeing (memory is still in use).
#define CONTAINER_FREE_EVENT(state, container) \
CONTAINER_FREE_STAT(state, container) \
CONTAINER_FREE_TRACE(state, container)
// Called on container destroy (memory is released to allocator).
#define CONTAINER_DESTROY_EVENT(state, container) \
CONTAINER_DESTROY_STAT(state, container) \
@@ -370,9 +383,9 @@ struct MemoryState {
OBJECT_FREE_STAT(state, size, object) \
OBJECT_FREE_TRACE(state, object)
// Reference in memory is being updated.
#define UPDATE_REF_EVENT(state, oldRef, newRef, slot) \
UPDATE_REF_STAT(state, oldRef, newRef, slot) \
UPDATE_REF_TRACE(state, oldRef, newRef, slot)
#define UPDATE_REF_EVENT(state, oldRef, newRef, slot, stack) \
UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) \
UPDATE_REF_TRACE(state, oldRef, newRef, slot, stack)
// Infomation shall be printed as worker is exiting.
#define PRINT_EVENT(state) \
PRINT_STAT(state)
@@ -494,7 +507,7 @@ void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) {
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(body) + typeInfo->objOffsets_[index]);
UpdateRef(location, nullptr);
ZeroHeapRef(location);
}
}
@@ -574,9 +587,9 @@ inline void scheduleDestroyContainer(MemoryState* state, ContainerHeader* contai
processFinalizerQueue(state);
}
#else
atomicAdd(&allocCount, -1);
CONTAINER_DESTROY_EVENT(state, container)
konanFreeMemory(container);
atomicAdd(&allocCount, -1);
CONTAINER_DESTROY_EVENT(state, container);
#endif
}
@@ -585,7 +598,6 @@ inline void scheduleDestroyContainer(MemoryState* state, ContainerHeader* contai
template <bool Atomic>
inline void IncrementRC(ContainerHeader* container) {
container->incRefCount<Atomic>();
UPDATE_ADDREF_STAT(memoryState, container, Atomic);
}
template <bool Atomic, bool UseCycleCollector>
@@ -593,7 +605,6 @@ inline void DecrementRC(ContainerHeader* container) {
if (container->decRefCount<Atomic>() == 0) {
FreeContainer(container);
}
UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false);
}
#else // USE_GC
@@ -606,13 +617,11 @@ template <bool Atomic>
inline void IncrementRC(ContainerHeader* container) {
container->incRefCount<Atomic>();
container->setColorUnlessGreen(CONTAINER_TAG_GC_BLACK);
UPDATE_ADDREF_STAT(memoryState, container, Atomic);
}
template <bool Atomic, bool UseCycleCollector>
inline void DecrementRC(ContainerHeader* container) {
if (container->decRefCount<Atomic>() == 0) {
UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false);
FreeContainer(container);
} else if (UseCycleCollector) { // Possible root.
RuntimeAssert(!Atomic, "Cycle collector shalln't be used with shared objects yet");
@@ -623,7 +632,6 @@ inline void DecrementRC(ContainerHeader* container) {
// Also do not use cycle collector for provable acyclic objects.
int color = container->color();
if (color != CONTAINER_TAG_GC_PURPLE && color != CONTAINER_TAG_GC_GREEN) {
UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, true);
container->setColorAssertIfGreen(CONTAINER_TAG_GC_PURPLE);
if (!container->buffered()) {
container->setBuffered();
@@ -633,8 +641,6 @@ inline void DecrementRC(ContainerHeader* container) {
GarbageCollect();
}
}
} else {
UPDATE_RELEASEREF_STAT(memoryState, container, Atomic, false);
}
}
}
@@ -866,7 +872,7 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) {
auto* childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (Shareable(childContainer)) {
UpdateRef(location, nullptr);
ZeroHeapRef(location);
} else {
toVisit.push_front(childContainer);
}
@@ -877,23 +883,53 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) {
}
#endif
inline void AddRef(ContainerHeader* header) {
inline bool needAtomicAccess(ContainerHeader* container) {
return container->shareable();
}
inline bool canBeCyclic(ContainerHeader* container) {
if (container->refCount() == 1) return false;
if (container->color() == CONTAINER_TAG_GC_GREEN) return false;
return true;
}
ALWAYS_INLINE inline void addRef(ContainerHeader* container) {
// Looking at container type we may want to skip AddRef() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
switch (container->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK:
break;
case CONTAINER_TAG_NORMAL:
IncrementRC</* Atomic = */ false>(header);
IncrementRC</* Atomic = */ false>(container);
break;
/* case CONTAINER_TAG_FROZEN: case CONTAINER_TAG_ATOMIC: */
default:
IncrementRC</* Atomic = */ true>(header);
IncrementRC</* Atomic = */ true>(container);
break;
}
}
inline void ReleaseRef(ContainerHeader* header) {
ALWAYS_INLINE inline void AddHeapRef(ContainerHeader* container) {
UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 0);
addRef(container);
}
ALWAYS_INLINE inline void AddHeapRef(ObjHeader* header) {
auto* container = header->container();
if (container != nullptr) AddHeapRef(container);
}
ALWAYS_INLINE inline void AddStackRef(ContainerHeader* container) {
UPDATE_ADDREF_STAT(memoryState, container, needAtomicAccess(container), 1);
addRef(container);
}
ALWAYS_INLINE inline void AddStackRef(ObjHeader* header) {
auto* container = header->container();
if (container != nullptr) AddStackRef(container);
}
ALWAYS_INLINE inline void releaseRef(ContainerHeader* header) {
// Looking at container type we may want to skip ReleaseRef() totally
// (non-escaping stack objects, constant objects).
switch (header->tag()) {
@@ -909,6 +945,26 @@ inline void ReleaseRef(ContainerHeader* header) {
}
}
ALWAYS_INLINE inline void ReleaseHeapRef(ContainerHeader* container) {
UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 0);
releaseRef(container);
}
ALWAYS_INLINE inline void ReleaseStackRef(ContainerHeader* container) {
UPDATE_RELEASEREF_STAT(memoryState, container, needAtomicAccess(container), canBeCyclic(container), 1);
releaseRef(container);
}
ALWAYS_INLINE inline void ReleaseHeapRef(ObjHeader* header) {
auto* container = header->container();
if (container != nullptr) ReleaseHeapRef(container);
}
ALWAYS_INLINE inline void ReleaseStackRef(ObjHeader* header) {
auto* container = header->container();
if (container != nullptr) ReleaseStackRef(container);
}
// We use first slot as place to store frame-local arena container.
// TODO: create ArenaContainer object on the stack, so that we don't
// do two allocations per frame (ArenaContainer + actual container).
@@ -961,7 +1017,7 @@ void ObjHeader::destroyMetaObject(TypeInfo** location) {
*const_cast<const TypeInfo**>(location) = meta->typeInfo_;
if (meta->counter_ != nullptr) {
WeakReferenceCounterClear(meta->counter_);
UpdateRef(&meta->counter_, nullptr);
ZeroHeapRef(&meta->counter_);
}
#ifdef KONAN_OBJC_INTEROP
@@ -1029,8 +1085,6 @@ void FreeContainer(ContainerHeader* container) {
RuntimeAssert(container != nullptr, "this kind of container shalln't be freed");
auto state = memoryState;
CONTAINER_FREE_EVENT(state, container)
if (isAggregatingFrozenContainer(container)) {
FreeAggregatingFrozenContainer(container);
return;
@@ -1040,7 +1094,7 @@ void FreeContainer(ContainerHeader* container) {
// Now let's clean all object's fields in this container.
traverseContainerObjectFields(container, [](ObjHeader** location) {
UpdateRef(location, nullptr);
ZeroHeapRef(location);
});
// And release underlying memory.
@@ -1177,28 +1231,28 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun
return result;
}
inline void AddRef(const ObjHeader* object) {
inline void addRef(const ObjHeader* object) {
auto* container = object->container();
if (container != nullptr) {
MEMORY_LOG("AddRef on %p in %p\n", object, container)
AddRef(container);
addRef(container);
}
}
inline void ReleaseRef(const ObjHeader* object) {
inline void releaseRef(const ObjHeader* object) {
auto* container = object->container();
if (container != nullptr) {
MEMORY_LOG("ReleaseRef on %p in %p\n", object, container)
ReleaseRef(container);
releaseRef(container);
}
}
void AddRefFromAssociatedObject(const ObjHeader* object) {
AddRef(object);
AddHeapRef(const_cast<ObjHeader*>(object));
}
void ReleaseRefFromAssociatedObject(const ObjHeader* object) {
ReleaseRef(object);
ReleaseHeapRef(const_cast<ObjHeader*>(object));
}
extern "C" {
@@ -1304,8 +1358,7 @@ OBJ_GETTER(InitInstance,
}
ObjHeader* object = AllocInstance(type_info, OBJ_RESULT);
MEMORY_LOG("Calling UpdateRef from InitInstance\n")
UpdateRef(location, object);
UpdateHeapRef(location, object);
#if KONAN_NO_EXCEPTIONS
ctor(object);
return object;
@@ -1314,8 +1367,8 @@ OBJ_GETTER(InitInstance,
ctor(object);
return object;
} catch (...) {
UpdateRef(OBJ_RESULT, nullptr);
UpdateRef(location, nullptr);
ZeroStackRef(OBJ_RESULT);
ZeroHeapRef(location);
throw;
}
#endif
@@ -1330,7 +1383,7 @@ OBJ_GETTER(InitSharedInstance,
RETURN_OBJ(value);
}
ObjHeader* object = AllocInstance(type_info, OBJ_RESULT);
UpdateRef(location, object);
UpdateHeapRef(location, object);
#if KONAN_NO_EXCEPTIONS
ctor(object);
FreezeSubgraph(object);
@@ -1341,8 +1394,8 @@ OBJ_GETTER(InitSharedInstance,
FreezeSubgraph(object);
return object;
} catch (...) {
UpdateRef(OBJ_RESULT, nullptr);
UpdateRef(location, nullptr);
ZeroStackRef(OBJ_RESULT);
ZeroHeapRef(location);
throw;
}
#endif
@@ -1360,37 +1413,57 @@ OBJ_GETTER(InitSharedInstance,
}
ObjHeader* object = AllocInstance(type_info, OBJ_RESULT);
RuntimeAssert(object->container()->normal() , "Shared object cannot be co-allocated");
MEMORY_LOG("Calling UpdateRef from InitSharedInstance\n")
UpdateRef(localLocation, object);
UpdateHeapRef(localLocation, object);
#if KONAN_NO_EXCEPTIONS
ctor(object);
FreezeSubgraph(object);
UpdateRef(location, object);
__sync_synchronize();
UpdateHeapRef(location, object);
synchronize();
return object;
#else
try {
ctor(object);
FreezeSubgraph(object);
UpdateRef(location, object);
__sync_synchronize();
UpdateHeapRef(location, object);
synchronize();
return object;
} catch (...) {
UpdateRef(OBJ_RESULT, nullptr);
UpdateRef(location, nullptr);
UpdateRef(localLocation, nullptr);
__sync_synchronize();
ZeroStackRef(OBJ_RESULT);
ZeroHeapRef(location);
ZeroHeapRef(localLocation);
synchronize();
throw;
}
#endif
#endif
}
void SetRef(ObjHeader** location, const ObjHeader* object) {
MEMORY_LOG("SetRef *%p: %p\n", location, object)
void SetHeapRef(ObjHeader** location, const ObjHeader* object) {
MEMORY_LOG("SetHeapRef *%p: %p\n", location, object)
UPDATE_REF_EVENT(memoryState, *location, object, location, 0);
*const_cast<const ObjHeader**>(location) = object;
if (object != nullptr)
AddRef(object);
AddHeapRef(const_cast<ObjHeader*>(object));
}
void ZeroHeapRef(ObjHeader** location) {
MEMORY_LOG("ZeroHeapRef %p\n", location)
auto* value = *location;
if (value != nullptr) {
UPDATE_REF_EVENT(memoryState, value, nullptr, location, 0);
*location = nullptr;
ReleaseHeapRef(value);
}
}
void ZeroStackRef(ObjHeader** location) {
MEMORY_LOG("ZeroStackRef %p\n", location)
auto* value = *location;
if (value != nullptr) {
UPDATE_REF_EVENT(memoryState, value, nullptr, location, 1);
*location = nullptr;
ReleaseStackRef(value);
}
}
ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) {
@@ -1406,60 +1479,71 @@ ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) {
return reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(&chunk->arena) | ARENA_BIT);
}
void UpdateRef(ObjHeader** location, const ObjHeader* object) {
ALWAYS_INLINE void updateRef(ObjHeader** location, const ObjHeader* object) {
RuntimeAssert(!isArenaSlot(location), "must not be a slot");
ObjHeader* old = *location;
UPDATE_REF_EVENT(memoryState, old, object, location)
if (old != object) {
if (object != nullptr) {
AddRef(object);
addRef(object);
}
*const_cast<const ObjHeader**>(location) = object;
if (reinterpret_cast<uintptr_t>(old) > 1) {
ReleaseRef(old);
releaseRef(old);
}
}
}
inline ObjHeader** slotAddressFor(ObjHeader** returnSlot, const ObjHeader* value) {
void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
UPDATE_REF_EVENT(memoryState, *location, object, location, 1);
updateRef(location, object);
}
void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) {
UPDATE_REF_EVENT(memoryState, *location, object, location, 0);
updateRef(location, object);
}
ALWAYS_INLINE inline ObjHeader** slotAddressFor(ObjHeader** returnSlot, const ObjHeader* value) {
if (!isArenaSlot(returnSlot)) return returnSlot;
// Not a subject of reference counting.
if (value == nullptr || !isRefCounted(value)) return nullptr;
return initedArena(asArenaSlot(returnSlot))->getSlot();
}
inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) {
ALWAYS_INLINE inline void updateReturnRefAdded(ObjHeader** returnSlot, const ObjHeader* value) {
returnSlot = slotAddressFor(returnSlot, value);
if (returnSlot == nullptr) return;
ObjHeader* old = *returnSlot;
UPDATE_REF_EVENT(memoryState, old, value, returnSlot, 1);
*const_cast<const ObjHeader**>(returnSlot) = value;
if (old != nullptr) {
ReleaseRef(old);
ReleaseStackRef(old);
}
}
void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* value) {
returnSlot = slotAddressFor(returnSlot, value);
if (returnSlot == nullptr) return;
UpdateRef(returnSlot, value);
UpdateStackRef(returnSlot, value);
}
void UpdateRefIfNull(ObjHeader** location, const ObjHeader* object) {
void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
if (object != nullptr) {
#if KONAN_NO_THREADS
ObjHeader* old = *location;
if (old == nullptr) {
AddRef(object);
AddHeapRef(const_cast<ObjHeader*>(object));
*const_cast<const ObjHeader**>(location) = object;
}
#else
AddRef(object);
AddHeapRef(const_cast<ObjHeader*>(object));
auto old = __sync_val_compare_and_swap(location, nullptr, const_cast<ObjHeader*>(object));
if (old != nullptr) {
// Failed to store, was not null.
ReleaseRef(object);
ReleaseHeapRef(const_cast<ObjHeader*>(object));
}
#endif
UPDATE_REF_EVENT(memoryState, old, object, location, 0);
}
}
@@ -1467,9 +1551,25 @@ void EnterFrame(ObjHeader** start, int parameters, int count) {
MEMORY_LOG("EnterFrame %p .. %p\n", start, start + count + parameters)
}
inline void releaseStackRefs(MemoryState* state, ObjHeader** start, int count) {
MEMORY_LOG("ReleaseStackRefs %p .. %p\n", start, start + count)
ObjHeader** current = start;
while (count-- > 0) {
ObjHeader* object = *current;
if (object != nullptr) {
UPDATE_REF_EVENT(memoryState, object, nullptr, current, 1);
ReleaseStackRef(object);
// Just for sanity, optional.
*current = nullptr;
}
current++;
}
}
void LeaveFrame(ObjHeader** start, int parameters, int count) {
MEMORY_LOG("LeaveFrame %p .. %p\n", start, start + count + parameters)
ReleaseRefs(start + parameters + kFrameOverlaySlots, count - kFrameOverlaySlots - parameters);
auto* state = memoryState;
releaseStackRefs(state, start + parameters + kFrameOverlaySlots, count - kFrameOverlaySlots - parameters);
if (*start != nullptr) {
auto arena = initedArena(start);
MEMORY_LOG("LeaveFrame: free arena %p\n", arena)
@@ -1479,20 +1579,6 @@ void LeaveFrame(ObjHeader** start, int parameters, int count) {
}
}
void ReleaseRefs(ObjHeader** start, int count) {
MEMORY_LOG("ReleaseRefs %p .. %p\n", start, start + count)
ObjHeader** current = start;
auto state = memoryState;
while (count-- > 0) {
ObjHeader* object = *current;
if (object != nullptr) {
ReleaseRef(object);
// Just for sanity, optional.
*current = nullptr;
}
current++;
}
}
#if USE_GC
@@ -1599,14 +1685,14 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) {
KNativePtr CreateStablePointer(KRef any) {
if (any == nullptr) return nullptr;
AddRef(any);
AddHeapRef(any);
return reinterpret_cast<KNativePtr>(any);
}
void DisposeStablePointer(KNativePtr pointer) {
if (pointer == nullptr) return;
KRef ref = reinterpret_cast<KRef>(pointer);
ReleaseRef(ref);
ReleaseHeapRef(ref);
}
OBJ_GETTER(DerefStablePointer, KNativePtr pointer) {
@@ -1615,11 +1701,10 @@ OBJ_GETTER(DerefStablePointer, KNativePtr pointer) {
}
OBJ_GETTER(AdoptStablePointer, KNativePtr pointer) {
#ifndef KONAN_NO_THREADS
__sync_synchronize();
#endif
synchronize();
KRef ref = reinterpret_cast<KRef>(pointer);
UpdateRef(OBJ_RESULT, nullptr);
// TODO: as it is called from C runtime,
ZeroHeapRef(OBJ_RESULT);
// Somewhat hacky.
*OBJ_RESULT = ref;
return ref;
@@ -1909,17 +1994,17 @@ void MutationCheck(ObjHeader* obj) {
if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj);
}
OBJ_GETTER(SwapRefLocked,
OBJ_GETTER(SwapHeapRefLocked,
ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) {
lock(spinlock);
ObjHeader* oldValue = *location;
// We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock.
if (oldValue == expectedValue) {
SetRef(location, newValue);
SetHeapRef(location, newValue);
} else {
// We create an additional reference to the [oldValue] in the return slot.
if (oldValue != nullptr && isRefCounted(oldValue)) {
AddRef(oldValue);
AddHeapRef(oldValue);
}
}
unlock(spinlock);
@@ -1929,14 +2014,14 @@ OBJ_GETTER(SwapRefLocked,
return oldValue;
}
void SetRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) {
void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) {
lock(spinlock);
ObjHeader* oldValue = *location;
// We do not use UpdateRef() here to avoid having ReleaseRef() on old value under the lock.
SetRef(location, newValue);
SetHeapRef(location, newValue);
unlock(spinlock);
if (oldValue != nullptr)
ReleaseRef(oldValue);
ReleaseHeapRef(oldValue);
}
OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) {
@@ -1944,7 +2029,7 @@ OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) {
ObjHeader* value = *location;
// We do not use UpdateRef() here to avoid having ReleaseRef() on return slot under the lock.
if (value != nullptr)
AddRef(value);
AddHeapRef(value);
unlock(spinlock);
updateReturnRefAdded(OBJ_RESULT, value);
return value;
@@ -1995,7 +2080,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what
}
if (!acyclic) return false;
}
UpdateRef(reinterpret_cast<ObjHeader**>(
UpdateHeapRef(reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(where) + where->type_info()->objOffsets_[index]), what);
// Fence on updated location?
return true;
runtime/src/main/cpp/Memory.h
+19 -15
View File
@@ -492,7 +492,7 @@ void WeakReferenceCounterClear(ObjHeader* counter);
// Reference management scheme we use assumes significant degree of flexibility, so that
// one could implement either pure reference counting scheme, or tracing collector without
// much ado.
// Most important primitive is UpdateRef() API, which modifies location to use new
// Most important primitive is Update*Ref() API, which modifies location to use new
// object reference. In pure reference counted scheme it will check old value,
// decrement reference, increment counter on the new value, and store it into the field.
// In tracing collector-like scheme, only field updates counts, and all other operations are
@@ -508,23 +508,27 @@ void WeakReferenceCounterClear(ObjHeader* counter);
// in intermediate frames when throwing
//
// Sets location.
void SetRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Updates location.
void UpdateRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Sets heap location.
void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Zeroes heap location.
void ZeroHeapRef(ObjHeader** location) RUNTIME_NOTHROW;
// Zeroes stack location.
void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW;
// Updates stack location.
void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Updates heap/static data location.
void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Updates location if it is null, atomically.
void UpdateRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
// Updates reference in return slot.
void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW;
// Compares and swaps reference with taken lock.
OBJ_GETTER(SwapRefLocked,
OBJ_GETTER(SwapHeapRefLocked,
ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW;
// Sets reference with taken lock.
void SetRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW;
void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW;
// Reads reference with taken lock.
OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW;
// Optimization: release all references in range.
void ReleaseRefs(ObjHeader** start, int count) RUNTIME_NOTHROW;
// Called on frame enter, if it has object slots.
void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
// Called on frame leave, if it has object slots.
@@ -563,16 +567,16 @@ class ObjHolder {
ObjHolder() : obj_(nullptr) {}
explicit ObjHolder(const ObjHeader* obj) {
::SetRef(&obj_, obj);
::SetHeapRef(&obj_, obj);
}
~ObjHolder() {
::UpdateRef(&obj_, nullptr);
::ZeroHeapRef(&obj_);
}
ObjHeader* obj() { return obj_; }
const ObjHeader* obj() const { return obj_; }
ObjHeader** slot() { return &obj_; }
void clear() { ::UpdateRef(&obj_, nullptr); }
void clear() { ::ZeroHeapRef(&obj_); }
private:
ObjHeader* obj_;
@@ -586,7 +590,7 @@ class KRefSharedHolder {
}
inline void init(ObjHeader* obj) {
SetRef(slotToInit(), obj);
SetHeapRef(slotToInit(), obj);
}
inline ObjHeader* ref() const {
@@ -596,7 +600,7 @@ class KRefSharedHolder {
inline void dispose() {
verifyRefOwner();
UpdateRef(&obj_, nullptr);
ZeroHeapRef(&obj_);
}
private:
runtime/src/main/cpp/Weak.cpp
+1 -1
View File
@@ -65,7 +65,7 @@ OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) {
ObjHolder counterHolder;
// Cast unneeded, just to emphasize we store an object reference as void*.
ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast<void*>(referred), counterHolder.slot());
UpdateRefIfNull(&meta->counter_, counter);
UpdateHeapRefIfNull(&meta->counter_, counter);
}
RETURN_OBJ(meta->counter_);
}
runtime/src/main/cpp/Worker.cpp
+1 -2
View File
@@ -67,9 +67,8 @@ KNativePtr transfer(KRef object, KInt mode) {
case UNCHECKED:
if (!ClearSubgraphReferences(object, mode == CHECKED)) {
// Release reference to the object, as it is not being managed by ObjHolder.
UpdateRef(&object, nullptr);
ZeroHeapRef(&object);
ThrowWorkerInvalidState();
return nullptr;
}
return object;
}
runtime/src/main/kotlin/kotlin/native/internal/test/Launcher.kt
+4
View File
@@ -31,3 +31,7 @@ fun worker(args: Array<String>) {
worker.requestTermination().result
exitProcess(result)
}
fun mainNoExit(args: Array<String>) {
testLauncherEntryPoint(args)
}