[K/N] Make GlobalData be initialized lazily ^KT-64313

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt

@@ -78,6 +78,8 @@ object BinaryOptions : BinaryOptionRegistry() {
     val packFields by booleanOption()
 
     val cInterfaceMode by option<CInterfaceGenerationMode>()
+
+    val globalDataLazyInit by booleanOption()
 }
 
 open class BinaryOption<T : Any>(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -262,6 +262,10 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         }
     } ?: target.supportsSignposts
 
+    val globalDataLazyInit: Boolean by lazy {
+        configuration.get(BinaryOptions.globalDataLazyInit) ?: true
+    }
+
     init {
         if (!platformManager.isEnabled(target)) {
             error("Target ${target.visibleName} is not available on the ${HostManager.hostName} host")

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2819,6 +2819,7 @@ internal class CodeGeneratorVisitor(
         overrideRuntimeGlobal("Kotlin_objcDisposeOnMain", llvm.constInt32(if (context.config.objcDisposeOnMain) 1 else 0))
         overrideRuntimeGlobal("Kotlin_objcDisposeWithRunLoop", llvm.constInt32(if (context.config.objcDisposeWithRunLoop) 1 else 0))
         overrideRuntimeGlobal("Kotlin_enableSafepointSignposts", llvm.constInt32(if (context.config.enableSafepointSignposts) 1 else 0))
+        overrideRuntimeGlobal("Kotlin_globalDataLazyInit", llvm.constInt32(if (context.config.globalDataLazyInit) 1 else 0))
     }
 
     //-------------------------------------------------------------------------//

kotlin-native/runtime/src/externalCallsChecker/impl/cpp/CallsChecker.cpp

@@ -14,7 +14,6 @@
 #include "Porting.h"
 #include "StackTrace.hpp"
 #include "ThreadData.hpp"
-#include "ThreadRegistry.hpp"
 #include "ExecFormat.h"
 
 using namespace kotlin;
@@ -346,7 +345,7 @@ extern "C" RUNTIME_NOTHROW RUNTIME_NODEBUG void Kotlin_mm_checkStateAtExternalFu
     if (reinterpret_cast<int64_t>(calleePtr) == MSG_SEND_TO_NULL) return; // objc_sendMsg called on nil, it does nothing, so it's ok
     if (ignoreGuardsCount != 0) return;
     if (konan::isOnThreadExitNotSetOrAlreadyStarted()) return;
-    if (!mm::ThreadRegistry::Instance().IsCurrentThreadRegistered()) return;
+    if (!mm::IsCurrentThreadRegistered()) return;
     CallsCheckerIgnoreGuard recursiveGuard;
 
     auto actualState = GetThreadState();

kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp

@@ -34,6 +34,7 @@ RUNTIME_WEAK int32_t Kotlin_mimallocUseCompaction = 0;
 RUNTIME_WEAK int32_t Kotlin_objcDisposeOnMain = 0;
 RUNTIME_WEAK int32_t Kotlin_objcDisposeWithRunLoop = 1;
 RUNTIME_WEAK int32_t Kotlin_enableSafepointSignposts = 0;
+RUNTIME_WEAK int32_t Kotlin_globalDataLazyInit = 1;
 
 ALWAYS_INLINE compiler::DestroyRuntimeMode compiler::destroyRuntimeMode() noexcept {
     return static_cast<compiler::DestroyRuntimeMode>(Kotlin_destroyRuntimeMode);
@@ -93,3 +94,7 @@ ALWAYS_INLINE bool compiler::objcDisposeWithRunLoop() noexcept {
 ALWAYS_INLINE bool compiler::enableSafepointSignposts() noexcept {
     return Kotlin_enableSafepointSignposts != 0;
 }
+
+ALWAYS_INLINE bool compiler::globalDataLazyInit() noexcept {
+    return Kotlin_globalDataLazyInit != 0;
+}

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -115,6 +115,7 @@ bool mimallocUseCompaction() noexcept;
 bool objcDisposeOnMain() noexcept;
 bool objcDisposeWithRunLoop() noexcept;
 bool enableSafepointSignposts() noexcept;
+bool globalDataLazyInit() noexcept;
 
 #ifdef KONAN_ANDROID
 bool printToAndroidLogcat() noexcept;

kotlin-native/runtime/src/main/cpp/ManuallyScoped.hpp

@@ -18,7 +18,7 @@ class ManuallyScoped : private Pinned {
 public:
     // Construct T
     template <typename... Args>
-    void construct(Args&&... args) noexcept(noexcept(T(std::forward<Args>(args)...))) {
+    void construct(Args&&... args) noexcept(std::is_nothrow_constructible_v<T, Args...>) {
         new (impl()) T(std::forward<Args>(args)...);
     }
 

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -236,7 +236,7 @@ extern "C" {
 
 struct MemoryState;
 
-MemoryState* InitMemory(bool firstRuntime);
+MemoryState* InitMemory();
 void DeinitMemory(MemoryState*, bool destroyRuntime);
 void RestoreMemory(MemoryState*);
 void ClearMemoryForTests(MemoryState*);
@@ -589,6 +589,8 @@ private:
 
 extern const bool kSupportsMultipleMutators;
 
+void initGlobalMemory() noexcept;
+
 void StartFinalizerThreadIfNeeded() noexcept;
 bool FinalizersThreadIsRunning() noexcept;
 

kotlin-native/runtime/src/main/cpp/Runtime.cpp

@@ -91,56 +91,24 @@ RuntimeState* initRuntime() {
   RuntimeCheck(!isValidRuntime(), "No active runtimes allowed");
   ::runtimeState = result;
 
-  bool firstRuntime = false;
-  // We set this guard in the `switch` below, after memory initialization.
-  kotlin::ThreadStateGuard stateGuard;
-  switch (kotlin::compiler::destroyRuntimeMode()) {
-      case kotlin::compiler::DestroyRuntimeMode::kLegacy:
-          compareAndSwap(&globalRuntimeStatus, kGlobalRuntimeUninitialized, kGlobalRuntimeRunning);
-          result->memoryState = InitMemory(false); // The argument will be ignored for legacy DestroyRuntimeMode
-          // Switch thread state because worker and globals inits require the runnable state.
-          // This call may block if GC requested suspending threads.
-          stateGuard = kotlin::ThreadStateGuard(result->memoryState, kotlin::ThreadState::kRunnable);
-          result->worker = WorkerInit(result->memoryState);
-          firstRuntime = atomicAdd(&aliveRuntimesCount, 1) == 1;
-          if (!kotlin::kSupportsMultipleMutators && !firstRuntime) {
-              konan::consoleErrorf("This GC implementation does not support multiple mutator threads.");
-              std::abort();
-          }
-          break;
-      case kotlin::compiler::DestroyRuntimeMode::kOnShutdown:
-          // First update `aliveRuntimesCount` and then update `globalRuntimeStatus`, for synchronization with
-          // runtime shutdown, which does it the other way around.
-          atomicAdd(&aliveRuntimesCount, 1);
-          auto lastStatus = compareAndSwap(&globalRuntimeStatus, kGlobalRuntimeUninitialized, kGlobalRuntimeRunning);
-          if (Kotlin_forceCheckedShutdown()) {
-              RuntimeAssert(lastStatus != kGlobalRuntimeShutdown, "Kotlin runtime was shut down. Cannot create new runtimes.");
-          }
-          firstRuntime = lastStatus == kGlobalRuntimeUninitialized;
-          if (!kotlin::kSupportsMultipleMutators && !firstRuntime) {
-              konan::consoleErrorf("This GC implementation does not support multiple mutator threads.");
-              std::abort();
-          }
-          result->memoryState = InitMemory(firstRuntime);
-          // Switch thread state because worker and globals inits require the runnable state.
-          // This call may block if GC requested suspending threads.
-          stateGuard = kotlin::ThreadStateGuard(result->memoryState, kotlin::ThreadState::kRunnable);
-          result->worker = WorkerInit(result->memoryState);
-  }
+  RuntimeAssert(compiler::destroyRuntimeMode() == compiler::DestroyRuntimeMode::kOnShutdown, "Legacy mode is not supported");
+
+  // First update `aliveRuntimesCount` and then update `globalRuntimeStatus`, for synchronization with
+  // runtime shutdown, which does it the other way around.
+  atomicAdd(&aliveRuntimesCount, 1);
+
+  bool firstRuntime = initializeGlobalRuntimeIfNeeded();
+  result->memoryState = InitMemory();
+  // Switch thread state because worker and globals inits require the runnable state.
+  // This call may block if GC requested suspending threads.
+  ThreadStateGuard stateGuard(result->memoryState, kotlin::ThreadState::kRunnable);
+  result->worker = WorkerInit(result->memoryState);
 
   InitOrDeinitGlobalVariables(ALLOC_THREAD_LOCAL_GLOBALS, result->memoryState);
   CommitTLSStorage(result->memoryState);
   // Keep global variables in state as well.
   if (firstRuntime) {
-    konan::consoleInit();
-    if (compiler::objcDisposeOnMain()) {
-      kotlin::initializeMainQueueProcessor();
-    }
-#if KONAN_OBJC_INTEROP
-    Kotlin_ObjCExport_initialize();
-#endif
     InitOrDeinitGlobalVariables(INIT_GLOBALS, result->memoryState);
-    logging::OnRuntimeInit();
   }
   InitOrDeinitGlobalVariables(INIT_THREAD_LOCAL_GLOBALS, result->memoryState);
   RuntimeAssert(result->status == RuntimeStatus::kUninitialized, "Runtime must still be in the uninitialized state");
@@ -192,6 +160,26 @@ void Kotlin_deinitRuntimeCallback(void* argument) {
 
 }  // namespace
 
+bool kotlin::initializeGlobalRuntimeIfNeeded() noexcept {
+    auto lastStatus = compareAndSwap(&globalRuntimeStatus, kGlobalRuntimeUninitialized, kGlobalRuntimeRunning);
+    if (Kotlin_forceCheckedShutdown()) {
+        RuntimeAssert(lastStatus != kGlobalRuntimeShutdown, "Kotlin runtime was shut down. Cannot create new runtimes.");
+    }
+    if (lastStatus != kGlobalRuntimeUninitialized)
+        return false;
+
+    konan::consoleInit();
+    logging::OnRuntimeInit();
+    initGlobalMemory();
+    if (compiler::objcDisposeOnMain()) {
+        initializeMainQueueProcessor();
+    }
+#if KONAN_OBJC_INTEROP
+    Kotlin_ObjCExport_initialize();
+#endif
+    return true;
+}
+
 extern "C" {
 
 RUNTIME_NOTHROW void AppendToInitializersTail(InitNode *next) {

kotlin-native/runtime/src/main/cpp/Runtime.h

@@ -44,4 +44,12 @@ bool Kotlin_forceCheckedShutdown();
 #ifdef __cplusplus
 } // extern "C"
 #endif
+
+namespace kotlin {
+
+// Returns `true` if initialized.
+bool initializeGlobalRuntimeIfNeeded() noexcept;
+
+}
+
 #endif // RUNTIME_RUNTIME_H

kotlin-native/runtime/src/main/cpp/TestSupport.hpp

@@ -23,7 +23,7 @@ constexpr int kDefaultThreadCount = 10;
 constexpr int kDefaultThreadCount = 100;
 #endif
 
-inline MemoryState* InitMemoryForTests() { return InitMemory(false); }
+inline MemoryState* InitMemoryForTests() { return InitMemory(); }
 void DeinitMemoryForTests(MemoryState* memoryState);
 
 // Scopely initializes the memory subsystem of the current thread for tests.

kotlin-native/runtime/src/mm/cpp/GlobalData.cpp

@@ -5,9 +5,97 @@
 
 #include "GlobalData.hpp"
 
+#include <condition_variable>
+#include <mutex>
+
+#include "CompilerConstants.hpp"
+#include "Porting.h"
+
 using namespace kotlin;
 
-mm::GlobalData::GlobalData() = default;
+namespace {
+
+enum class InitState {
+    kUninitialized,
+    kInitializing,
+    kInitialized,
+};
+
+const char* initStateToString(InitState state) noexcept {
+    switch (state) {
+        case InitState::kUninitialized: return "uninitialized";
+        case InitState::kInitializing: return "initializing";
+        case InitState::kInitialized: return "initialized";
+    }
+}
+
+std::atomic<InitState> globalDataInitState = InitState::kUninitialized;
+std::atomic<std::thread::id> globalDataInitializingThread{};
+ManuallyScoped<mm::GlobalData> globalDataInstance{};
+
+void constructGlobalDataInstance() noexcept {
+    auto initialState = InitState::kUninitialized;
+    globalDataInitState.compare_exchange_strong(initialState, InitState::kInitializing, std::memory_order_acq_rel);
+    RuntimeAssert(initialState == InitState::kUninitialized, "Expected state %s, but was %s", initStateToString(InitState::kUninitialized), initStateToString(initialState));
+    globalDataInitializingThread.store(std::this_thread::get_id(), std::memory_order_relaxed);
+
+    globalDataInstance.construct();
+
+    auto initializingState = InitState::kInitializing;
+    globalDataInitState.compare_exchange_strong(initializingState, InitState::kInitialized, std::memory_order_acq_rel);
+    RuntimeAssert(initializingState == InitState::kInitializing, "Expected state %s, but was %s", initStateToString(InitState::kInitializing), initStateToString(initializingState));
+}
+
+[[maybe_unused]] struct GlobalDataEagerInit {
+    GlobalDataEagerInit() noexcept {
+        if (!compiler::globalDataLazyInit()) {
+            constructGlobalDataInstance();
+        }
+    }
+} globalDataEagerInit;
+
+std::mutex globalDataLazyInitMutex;
+std::condition_variable globalDataLazyInitCV;
+
+}
+
+// static
+mm::GlobalData& mm::GlobalData::Instance() noexcept {
+    if (compiler::runtimeAssertsEnabled()) {
+        auto s = globalDataInitState.load(std::memory_order_relaxed);
+        auto initializingThread = globalDataInitializingThread.load(std::memory_order_relaxed);
+        RuntimeAssert(s == InitState::kInitialized || initializingThread == std::this_thread::get_id(), "Expected state %s, but was %s.", initStateToString(InitState::kInitialized), initStateToString(s));
+    }
+    return *globalDataInstance;
+}
+
+// static
+void mm::GlobalData::init() noexcept {
+    if (compiler::globalDataLazyInit()) {
+        std::unique_lock guard{globalDataLazyInitMutex};
+        constructGlobalDataInstance();
+        guard.unlock();
+        globalDataLazyInitCV.notify_all();
+    }
+}
+
+// static
+void mm::GlobalData::waitInitialized() noexcept {
+    if (compiler::globalDataLazyInit()) {
+        if (globalDataInitState.load(std::memory_order_acquire) == InitState::kInitialized) {
+            return;
+        }
+        if (compiler::runtimeAssertsEnabled()) {
+            auto initializingThread = globalDataInitializingThread.load(std::memory_order_relaxed);
+            RuntimeAssert(initializingThread != std::this_thread::get_id(), "A thread that initialized global data cannot be waiting for its initialization");
+        }
+        std::unique_lock guard{globalDataLazyInitMutex};
+        globalDataLazyInitCV.wait(guard, []() noexcept {
+            return globalDataInitState.load(std::memory_order_relaxed) == InitState::kInitialized;
+        });
+    }
+}
+
+mm::GlobalData::GlobalData() noexcept = default;
 
 // static
-mm::GlobalData mm::GlobalData::instance_ [[clang::no_destroy]];

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -7,13 +7,14 @@
 #define RUNTIME_MM_GLOBAL_DATA_H
 
 #include "Allocator.hpp"
-#include "GlobalsRegistry.hpp"
+#include "AppStateTracking.hpp"
 #include "GC.hpp"
 #include "GCScheduler.hpp"
+#include "GlobalsRegistry.hpp"
+#include "ManuallyScoped.hpp"
 #include "SpecialRefRegistry.hpp"
 #include "ThreadRegistry.hpp"
 #include "Utils.hpp"
-#include "AppStateTracking.hpp"
 
 namespace kotlin {
 namespace mm {
@@ -21,7 +22,11 @@ namespace mm {
 // Global (de)initialization is undefined in C++. Use single global singleton to define it for simplicity.
 class GlobalData : private Pinned {
 public:
-    static GlobalData& Instance() noexcept { return instance_; }
+    static GlobalData& Instance() noexcept;
+
+    // init() can only be called once.
+    static void init() noexcept;
+    static void waitInitialized() noexcept;
 
     ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; }
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
@@ -32,11 +37,10 @@ public:
     AppStateTracking& appStateTracking() noexcept { return appStateTracking_; }
 
 private:
-    GlobalData();
-    ~GlobalData() = delete;
+    friend class ManuallyScoped<GlobalData>;
 
-    // This `GlobalData` is never destroyed.
-    static GlobalData instance_;
+    GlobalData() noexcept;
+    ~GlobalData() = delete;
 
     ThreadRegistry threadRegistry_;
     AppStateTracking appStateTracking_;

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -95,10 +95,15 @@ ALWAYS_INLINE bool isShareable(const ObjHeader* obj) {
     return true;
 }
 
-extern "C" MemoryState* InitMemory(bool firstRuntime) {
+extern "C" MemoryState* InitMemory() {
+    mm::GlobalData::waitInitialized();
     return mm::ToMemoryState(mm::ThreadRegistry::Instance().RegisterCurrentThread());
 }
 
+void kotlin::initGlobalMemory() noexcept {
+    mm::GlobalData::init();
+}
+
 extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) {
     // We need the native state to avoid a deadlock on unregistering the thread.
     // The deadlock is possible if we are in the runnable state and the GC already locked
@@ -548,7 +553,7 @@ MemoryState* kotlin::mm::GetMemoryState() noexcept {
 }
 
 bool kotlin::mm::IsCurrentThreadRegistered() noexcept {
-    return ThreadRegistry::Instance().IsCurrentThreadRegistered();
+    return ThreadRegistry::IsCurrentThreadRegistered();
 }
 
 ALWAYS_INLINE kotlin::CalledFromNativeGuard::CalledFromNativeGuard(bool reentrant) noexcept : reentrant_(reentrant) {

kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp

@@ -46,7 +46,7 @@ public:
     }
     Node* CurrentThreadDataNodeOrNull() const noexcept { return currentThreadDataNode_; }
 
-    bool IsCurrentThreadRegistered() const noexcept { return currentThreadDataNode_ != nullptr; }
+    static bool IsCurrentThreadRegistered() noexcept { return currentThreadDataNode_ != nullptr; }
 
     static void ClearCurrentThreadData() { currentThreadDataNode_ = nullptr; }
 

kotlin-native/runtime/src/test_support/cpp/TestLauncher.cpp

@@ -6,6 +6,8 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "Runtime.h"
+
 extern "C" void Kotlin_TestSupport_AssertClearGlobalState();
 
 namespace {
@@ -27,5 +29,7 @@ int main(int argc, char** argv) {
     // Googletest takes ownership of the registered environment object.
     testing::AddGlobalTestEnvironment(new GlobalStateChecker());
 
+    kotlin::initializeGlobalRuntimeIfNeeded();
+
     return RUN_ALL_TESTS();
 }