From 5fc17ce5fafcb775b14d50e93419d2d0f82fe992 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Thu, 26 Nov 2020 11:25:25 +0300 Subject: [PATCH] Tweak TLS init/deinit (#4551) * Delete the entire TLS in one go * Put TLS allocation separately * Keep TLS in a single array --- .../kotlin/backend/konan/llvm/ContextUtils.kt | 1 - .../kotlin/backend/konan/llvm/IrToBitcode.kt | 30 ++--- .../runtime/src/legacymm/cpp/Memory.cpp | 116 ++++++++++++------ .../runtime/src/main/cpp/CompilerExport.cpp | 1 - kotlin-native/runtime/src/main/cpp/Memory.h | 6 +- .../runtime/src/main/cpp/Runtime.cpp | 11 +- kotlin-native/runtime/src/mm/cpp/Stubs.cpp | 6 +- 7 files changed, 101 insertions(+), 70 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 394e29ff712..7489df0b056 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -515,7 +515,6 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val throwExceptionFunction = importRtFunction("ThrowException") val appendToInitalizersTail = importRtFunction("AppendToInitializersTail") val addTLSRecord = importRtFunction("AddTLSRecord") - val clearTLSRecord = importRtFunction("ClearTLSRecord") val lookupTLS = importRtFunction("LookupTLS") val initRuntimeIfNeeded = importRtFunction("Kotlin_initRuntimeIfNeeded") val mutationCheck = importRtFunction("MutationCheck") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index f2044d3c6fc..613f0c9fa08 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -15,10 +15,8 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.ir.* import org.jetbrains.kotlin.backend.konan.llvm.coverage.LLVMCoverageInstrumentation -import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleFragmentImpl import org.jetbrains.kotlin.builtins.UnsignedType import org.jetbrains.kotlin.descriptors.Modality -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrStatement import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET @@ -37,7 +35,6 @@ import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.resolve.descriptorUtil.classId -import org.jetbrains.kotlin.resolve.descriptorUtil.module internal enum class FieldStorageKind { GLOBAL, // In the old memory model these are only accessible from the "main" thread. @@ -385,9 +382,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map 0) { - val memory = LLVMGetParam(initFunction, 1)!! - call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey, - Int32(context.llvm.tlsCount).llvm)) - } context.llvm.fileInitializers .forEach { irField -> if (irField.initializer?.expression !is IrConst<*>?) { @@ -436,11 +428,6 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map 0) { - val memory = LLVMGetParam(initFunction, 1)!! - call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey, - Int32(context.llvm.tlsCount).llvm)) - } context.llvm.fileInitializers .forEach { irField -> if (irField.initializer != null && irField.storageKind == FieldStorageKind.THREAD_LOCAL) { @@ -454,10 +441,11 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map 0) { val memory = LLVMGetParam(initFunction, 1)!! - call(context.llvm.clearTLSRecord, listOf(memory, context.llvm.tlsKey)) + call(context.llvm.addTLSRecord, listOf(memory, context.llvm.tlsKey, + Int32(context.llvm.tlsCount).llvm)) } ret(null) } diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index a6c365f417e..03bd85fdc84 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -131,7 +131,6 @@ typedef KStdUnorderedSet KRefSet; typedef KStdUnorderedMap KRefIntMap; typedef KStdDeque KRefDeque; typedef KStdDeque KRefListDeque; -typedef KStdUnorderedMap> KThreadLocalStorageMap; // A little hack that allows to enable -O2 optimizations // Prevents clang from replacing FrameOverlay struct @@ -584,15 +583,78 @@ private: } }; +namespace { + +class ThreadLocalStorage { +public: + using Key = void**; + + void Init() noexcept { map_ = konanConstructInstance(); } + + void Deinit() noexcept { + RuntimeAssert(map_->size() == 0, "Must be already cleared"); + konanDestructInstance(map_); + } + + void Add(Key key, int size) noexcept { + RuntimeAssert(storage_ == nullptr, "Storage must not be committed"); + auto it = map_->find(key); + if (it != map_->end()) { + RuntimeAssert(it->second.second == size, "Attempt to add TLS record with the same key and different size"); + return; + } + map_->emplace(key, std::make_pair(size_, size)); + size_ += size; + } + + void Commit() noexcept { + RuntimeAssert(storage_ == nullptr, "Cannot commit storage twice"); + storage_ = reinterpret_cast(konanAllocMemory(size_ * sizeof(KRef))); + } + + void Clear() noexcept { + RuntimeAssert(storage_ != nullptr, "Storage must be committed"); + for (int i = 0; i < size_; ++i) { + UpdateHeapRef(storage_ + i, nullptr); + } + konanFreeMemory(storage_); + map_->clear(); + } + + KRef* Lookup(Key key, int index) noexcept { + RuntimeAssert(storage_ != nullptr, "Storage must be committed"); + // In many cases there is only one module, so this is one element cache. + if (lastKey_ == key) { + return storage_ + lastOffset_ + index; + } + auto it = map_->find(key); + RuntimeAssert(it != map_->end(), "Must be there"); + int offset = it->second.first; + RuntimeAssert(offset + index < size_, "Out of bound in TLS access"); + lastKey_ = key; + lastOffset_ = offset; + return storage_ + offset + index; + } + +private: + using Map = KStdUnorderedMap>; + + Map* map_ = nullptr; + KRef* storage_ = nullptr; + int size_ = 0; + int lastOffset_ = 0; + Key lastKey_ = nullptr; +}; + +} // namespace + struct MemoryState { #if TRACE_MEMORY // Set of all containers. ContainerHeaderSet* containers; #endif - KThreadLocalStorageMap* tlsMap; - KRef* tlsMapLastStart; - void* tlsMapLastKey; + ThreadLocalStorage tls; #if USE_GC // Finalizer queue - linked list of containers scheduled for finalization. @@ -1995,7 +2057,7 @@ MemoryState* initMemory(bool firstRuntime) { memoryState->allocSinceLastGcThreshold = kMaxGcAllocThreshold; memoryState->gcErgonomics = true; #endif - memoryState->tlsMap = konanConstructInstance(); + memoryState->tls.Init(); memoryState->foreignRefManager = ForeignRefManager::create(); bool firstMemoryState = atomicAdd(&aliveMemoryStatesCount, 1) == 1; switch (Kotlin_getDestroyRuntimeMode()) { @@ -2051,8 +2113,7 @@ void deinitMemory(MemoryState* memoryState, bool destroyRuntime) { konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->roots); konanDestructInstance(memoryState->toRelease); - RuntimeAssert(memoryState->tlsMap->size() == 0, "Must be already cleared"); - konanDestructInstance(memoryState->tlsMap); + memoryState->tls.Deinit(); RuntimeAssert(memoryState->finalizerQueue == nullptr, "Finalizer queue must be empty"); RuntimeAssert(memoryState->finalizerQueueSize == 0, "Finalizer queue must be empty"); #endif // USE_GC @@ -3535,44 +3596,19 @@ void Kotlin_Any_share(ObjHeader* obj) { } RUNTIME_NOTHROW void AddTLSRecord(MemoryState* memory, void** key, int size) { - auto* tlsMap = memory->tlsMap; - auto it = tlsMap->find(key); - if (it != tlsMap->end()) { - RuntimeAssert(it->second.second == size, "Size must be consistent"); - return; - } - KRef* start = reinterpret_cast(konanAllocMemory(size * sizeof(KRef))); - tlsMap->emplace(key, std::make_pair(start, size)); + memory->tls.Add(key, size); } -RUNTIME_NOTHROW void ClearTLSRecord(MemoryState* memory, void** key) { - auto* tlsMap = memory->tlsMap; - auto it = tlsMap->find(key); - if (it != tlsMap->end()) { - KRef* start = it->second.first; - int count = it->second.second; - for (int i = 0; i < count; i++) { - UpdateHeapRef(start + i, nullptr); - } - konanFreeMemory(start); - tlsMap->erase(it); - } +RUNTIME_NOTHROW void CommitTLSStorage(MemoryState* memory) { + memory->tls.Commit(); +} + +RUNTIME_NOTHROW void ClearTLS(MemoryState* memory) { + memory->tls.Clear(); } RUNTIME_NOTHROW KRef* LookupTLS(void** key, int index) { - auto* state = memoryState; - auto* tlsMap = state->tlsMap; - // In many cases there is only one module, so this one element cache. - if (state->tlsMapLastKey == key) { - return state->tlsMapLastStart + index; - } - auto it = tlsMap->find(key); - RuntimeAssert(it != tlsMap->end(), "Must be there"); - RuntimeAssert(index < it->second.second, "Out of bound in TLS access"); - KRef* start = it->second.first; - state->tlsMapLastKey = key; - state->tlsMapLastStart = start; - return start + index; + return memoryState->tls.Lookup(key, index); } diff --git a/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp b/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp index 5263d6df773..a9341926cc2 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp @@ -35,7 +35,6 @@ void EnsureDeclarationsEmitted() { ensureUsed(EnterFrame); ensureUsed(LeaveFrame); ensureUsed(AddTLSRecord); - ensureUsed(ClearTLSRecord); ensureUsed(LookupTLS); ensureUsed(MutationCheck); ensureUsed(CheckLifetimesConstraint); diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index be5ab005796..2e06eb9e030 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -225,8 +225,10 @@ void FreezeSubgraph(ObjHeader* obj); void EnsureNeverFrozen(ObjHeader* obj); // Add TLS object storage, called by the generated code. void AddTLSRecord(MemoryState* memory, void** key, int size) RUNTIME_NOTHROW; -// Clear TLS object storage, called by the generated code. -void ClearTLSRecord(MemoryState* memory, void** key) RUNTIME_NOTHROW; +// Allocate storage for TLS. `AddTLSRecord` cannot be called after this. +void CommitTLSStorage(MemoryState* memory) RUNTIME_NOTHROW; +// Clear TLS object storage. +void ClearTLS(MemoryState* memory) RUNTIME_NOTHROW; // Lookup element in TLS object storage. ObjHeader** LookupTLS(void** key, int index) RUNTIME_NOTHROW; diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp index 9c81ded50df..42e6594c0a4 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp +++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp @@ -55,10 +55,11 @@ struct RuntimeState { RuntimeStatus status = RuntimeStatus::kUninitialized; }; +// Must be synchronized with IrToBitcode.kt enum { - INIT_GLOBALS = 0, - INIT_THREAD_LOCAL_GLOBALS = 1, - DEINIT_THREAD_LOCAL_GLOBALS = 2, + ALLOC_THREAD_LOCAL_GLOBALS = 0, + INIT_GLOBALS = 1, + INIT_THREAD_LOCAL_GLOBALS = 2, DEINIT_GLOBALS = 3 }; @@ -120,6 +121,8 @@ RuntimeState* initRuntime() { result->worker = WorkerInit(true); } + InitOrDeinitGlobalVariables(ALLOC_THREAD_LOCAL_GLOBALS, result->memoryState); + CommitTLSStorage(result->memoryState); // Keep global variables in state as well. if (firstRuntime) { konan::consoleInit(); @@ -148,7 +151,7 @@ void deinitRuntime(RuntimeState* state, bool destroyRuntime) { // Nothing to do. break; } - InitOrDeinitGlobalVariables(DEINIT_THREAD_LOCAL_GLOBALS, state->memoryState); + ClearTLS(state->memoryState); if (destroyRuntime) InitOrDeinitGlobalVariables(DEINIT_GLOBALS, state->memoryState); auto workerId = GetWorkerId(state->worker); diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index 57bd6ab0bed..f43418f967f 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -166,7 +166,11 @@ RUNTIME_NOTHROW void AddTLSRecord(MemoryState* memory, void** key, int size) { RuntimeCheck(false, "Unimplemented"); } -RUNTIME_NOTHROW void ClearTLSRecord(MemoryState* memory, void** key) { +RUNTIME_NOTHROW void CommitTLSStorage(MemoryState* memory) { + RuntimeCheck(false, "Unimplemented"); +} + +RUNTIME_NOTHROW void ClearTLS(MemoryState* memory) { RuntimeCheck(false, "Unimplemented"); }