[K/N] Rework GCScheduler to schedule GC on its own.

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt

@@ -53,7 +53,7 @@ private class CallsChecker(val context: Context, goodFunctions: List<String>) {
 
     private fun LLVMValueRef.getPossiblyExternalCalledFunction(): ExternalCallInfo? {
         fun isIndirectCallArgument(value: LLVMValueRef) = LLVMIsALoadInst(value) != null || LLVMIsAArgument(value) != null ||
-                LLVMIsAPHINode(value) != null || LLVMIsASelectInst(value) != null || LLVMIsACallInst(value) != null
+                LLVMIsAPHINode(value) != null || LLVMIsASelectInst(value) != null || LLVMIsACallInst(value) != null || LLVMIsAExtractElementInst(value) != null
 
         fun cleanCalledFunction(value: LLVMValueRef): ExternalCallInfo? {
             return when {
@@ -202,4 +202,4 @@ internal fun addFunctionsListSymbolForChecker(context: Context) {
             ?.setInitializer(Int32(functions.size))
             ?: throw IllegalStateException("$functionListSizeGlobal global not found")
     context.verifyBitCode()
-}
+}

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp

@@ -6,50 +6,119 @@
 #include "GCScheduler.hpp"
 
 #include "CompilerConstants.hpp"
+#include "GlobalData.hpp"
+#include "KAssert.h"
 #include "Porting.h"
+#include "RepeatedTimer.hpp"
+#include "ThreadRegistry.hpp"
+#include "ThreadData.hpp"
 
 using namespace kotlin;
 
-bool gc::GCScheduler::ThreadData::OnSafePointSlowPath() noexcept {
-    const auto result = onSafePoint_(allocatedBytes_, safePointsCounter_);
-    ClearCountersAndUpdateThresholds();
-    return result;
+namespace {
+
+class GCEmptySchedulerData : public gc::GCSchedulerData {
+    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {}
+    void OnPerformFullGC() noexcept override {}
+};
+
+class GCSchedulerDataWithTimer : public gc::GCSchedulerData {
+public:
+    explicit GCSchedulerDataWithTimer(gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
+        config_(config), scheduleGC_(std::move(scheduleGC)), timer_(std::chrono::microseconds(config_.regularGcIntervalUs), [this]() {
+            OnTimer();
+            return std::chrono::microseconds(config_.regularGcIntervalUs);
+        }) {}
+
+    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {
+        size_t allocatedBytes = threadData.allocatedBytes();
+        if (allocatedBytes > config_.allocationThresholdBytes) {
+            RuntimeAssert(static_cast<bool>(scheduleGC_), "scheduleGC_ cannot be empty");
+            scheduleGC_();
+        }
+    }
+
+    void OnPerformFullGC() noexcept override {}
+
+private:
+    void OnTimer() noexcept {
+        auto allThreadsAreNative = []() {
+            auto threadRegistryIter = mm::GlobalData::Instance().threadRegistry().LockForIter();
+            return std::all_of(threadRegistryIter.begin(), threadRegistryIter.end(), [](mm::ThreadData& thread) {
+                return thread.state() == ThreadState::kNative;
+            });
+        }();
+        // Don't run, if kotlin code is not being executed.
+        if (allThreadsAreNative) return;
+
+        // TODO: Probably makes sense to check memory usage of the process.
+        scheduleGC_();
+    }
+
+    gc::GCSchedulerConfig& config_;
+
+    std::function<void()> scheduleGC_;
+    RepeatedTimer timer_;
+};
+
+class GCSchedulerDataWithoutTimer : public gc::GCSchedulerData {
+public:
+    using CurrentTimeCallback = std::function<uint64_t()>;
+
+    GCSchedulerDataWithoutTimer(
+            gc::GCSchedulerConfig& config, std::function<void()> scheduleGC, CurrentTimeCallback currentTimeCallbackNs) noexcept :
+        config_(config),
+        currentTimeCallbackNs_(std::move(currentTimeCallbackNs)),
+        timeOfLastGcNs_(currentTimeCallbackNs_()),
+        scheduleGC_(std::move(scheduleGC)) {}
+
+    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {
+        size_t allocatedBytes = threadData.allocatedBytes();
+        if (allocatedBytes > config_.allocationThresholdBytes ||
+            currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs) {
+            RuntimeAssert(static_cast<bool>(scheduleGC_), "scheduleGC_ cannot be empty");
+            scheduleGC_();
+        }
+    }
+
+    void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); }
+
+private:
+    gc::GCSchedulerConfig& config_;
+    CurrentTimeCallback currentTimeCallbackNs_;
+
+    std::atomic<uint64_t> timeOfLastGcNs_;
+    std::function<void()> scheduleGC_;
+};
+
+} // namespace
+
+KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeEmptyGCSchedulerData() noexcept {
+    return ::make_unique<GCEmptySchedulerData>();
 }
 
-void gc::GCScheduler::ThreadData::ClearCountersAndUpdateThresholds() noexcept {
-    allocatedBytes_ = 0;
-    safePointsCounter_ = 0;
-
-    allocatedBytesThreshold_ = config_.allocationThresholdBytes;
-    safePointsCounterThreshold_ = config_.threshold;
+KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerDataWithTimer(
+        GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
+    return ::make_unique<GCSchedulerDataWithTimer>(config, std::move(scheduleGC));
 }
 
-gc::GCSchedulerConfig::GCSchedulerConfig() noexcept {
-    if (compiler::gcAggressive()) {
-        // TODO: Make it even more aggressive and run on a subset of backend.native tests.
-        threshold = 1000;
-        allocationThresholdBytes = 10000;
-        cooldownThresholdNs = 0;
+KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerDataWithoutTimer(
+        GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept {
+    return ::make_unique<GCSchedulerDataWithoutTimer>(config, std::move(scheduleGC), std::move(currentTimeCallbackNs));
+}
+
+KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerData(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
+    if (internal::useGCTimer()) {
+        return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC));
+    } else {
+        return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); });
     }
 }
 
-gc::GCScheduler::GCData::GCData(gc::GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept :
-    config_(config), currentTimeCallbackNs_(std::move(currentTimeCallbackNs)), timeOfLastGcNs_(currentTimeCallbackNs_()) {}
-
-bool gc::GCScheduler::GCData::OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept {
-    if (allocatedBytes > config_.allocationThresholdBytes) return true;
-
-    return currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs;
-}
-
-void gc::GCScheduler::GCData::OnPerformFullGC() noexcept {
-    timeOfLastGcNs_ = currentTimeCallbackNs_();
-}
-
-gc::GCScheduler::GCScheduler() noexcept : gcData_(config_, []() { return konan::getTimeNanos(); }) {}
-
-gc::GCScheduler::ThreadData gc::GCScheduler::NewThreadData() noexcept {
-    return ThreadData(config_, [this](size_t allocatedBytes, size_t safePointsCounter) {
-        return gcData().OnSafePoint(allocatedBytes, safePointsCounter);
-    });
+void gc::GCScheduler::SetScheduleGC(std::function<void()> scheduleGC) noexcept {
+    RuntimeAssert(static_cast<bool>(scheduleGC), "scheduleGC cannot be empty");
+    RuntimeAssert(!static_cast<bool>(scheduleGC_), "scheduleGC must not have been set");
+    scheduleGC_ = std::move(scheduleGC);
+    RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call");
+    gcData_ = internal::MakeGCSchedulerData(config_, scheduleGC_);
 }

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp

@@ -10,99 +10,162 @@
 #include <cinttypes>
 #include <cstddef>
 #include <functional>
+#include <utility>
 
+#include "CompilerConstants.hpp"
+#include "Types.h"
 #include "Utils.hpp"
 
 namespace kotlin {
 namespace gc {
 
+namespace internal {
+
+inline bool useGCTimer() noexcept {
+#if KONAN_NO_THREADS
+    return false;
+#else
+    // With aggressive mode we use safepoint counting to drive GC.
+    return !compiler::gcAggressive();
+#endif
+}
+
+} // namespace internal
+
 struct GCSchedulerConfig {
     std::atomic<size_t> threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine).
     std::atomic<size_t> allocationThresholdBytes = 10 * 1024 * 1024; // 10MiB by default.
     std::atomic<uint64_t> cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default.
     std::atomic<bool> autoTune = false;
+    std::atomic<uint64_t> regularGcIntervalUs = 200 * 1000; // 200 milliseconds by default.
 
-    GCSchedulerConfig() noexcept;
+    GCSchedulerConfig() noexcept {
+        if (compiler::gcAggressive()) {
+            // TODO: Make it even more aggressive and run on a subset of backend.native tests.
+            threshold = 1000;
+            allocationThresholdBytes = 10000;
+            cooldownThresholdNs = 0;
+        }
+    }
 };
 
-// TODO: Consider calling GC from the scheduler itself.
-class GCScheduler : private Pinned {
+class GCSchedulerThreadData;
+
+class GCSchedulerData {
 public:
-    class ThreadData {
-    public:
-        using OnSafePointCallback = std::function<bool(size_t, size_t)>;
+    virtual ~GCSchedulerData() = default;
 
-        static constexpr size_t kFunctionEpilogueWeight = 1;
-        static constexpr size_t kLoopBodyWeight = 1;
-        static constexpr size_t kExceptionUnwindWeight = 1;
+    // Called by different mutator threads.
+    virtual void OnSafePoint(GCSchedulerThreadData& threadData) noexcept = 0;
 
-        explicit ThreadData(GCSchedulerConfig& config, OnSafePointCallback onSafePoint) noexcept :
-            config_(config), onSafePoint_(std::move(onSafePoint)) {
-            ClearCountersAndUpdateThresholds();
-        }
+    // Always called by the GC thread.
+    virtual void OnPerformFullGC() noexcept = 0;
+};
 
-        // Should be called on encountering a safepoint.
-        bool OnSafePointRegular(size_t weight) noexcept {
+class GCSchedulerThreadData {
+public:
+    static constexpr size_t kFunctionEpilogueWeight = 1;
+    static constexpr size_t kLoopBodyWeight = 1;
+    static constexpr size_t kExceptionUnwindWeight = 1;
+
+    explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> onSafePoint) noexcept :
+        config_(config), onSafePoint_(std::move(onSafePoint)) {
+        ClearCountersAndUpdateThresholds();
+    }
+
+    // Should be called on encountering a safepoint.
+    void OnSafePointRegular(size_t weight) noexcept {
+        if (!internal::useGCTimer()) {
             safePointsCounter_ += weight;
             if (safePointsCounter_ < safePointsCounterThreshold_) {
-                return false;
+                return;
             }
-            return OnSafePointSlowPath();
+            OnSafePointSlowPath();
         }
+    }
 
-        // Should be called on encountering a safepoint placed by the allocator.
-        // TODO: Should this even be a safepoint (i.e. a place, where we suspend)?
-        bool OnSafePointAllocation(size_t size) noexcept {
-            allocatedBytes_ += size;
-            if (allocatedBytes_ < allocatedBytesThreshold_) {
-                return false;
-            }
-            return OnSafePointSlowPath();
+    // Should be called on encountering a safepoint placed by the allocator.
+    // TODO: Should this even be a safepoint (i.e. a place, where we suspend)?
+    void OnSafePointAllocation(size_t size) noexcept {
+        allocatedBytes_ += size;
+        if (allocatedBytes_ < allocatedBytesThreshold_) {
+            return;
         }
+        OnSafePointSlowPath();
+    }
 
-        void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); }
+    void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); }
 
-    private:
-        bool OnSafePointSlowPath() noexcept;
-        void ClearCountersAndUpdateThresholds() noexcept;
+    size_t allocatedBytes() const noexcept { return allocatedBytes_; }
 
-        GCSchedulerConfig& config_;
-        OnSafePointCallback onSafePoint_;
+    size_t safePointsCounter() const noexcept { return safePointsCounter_; }
 
-        size_t allocatedBytes_ = 0;
-        size_t allocatedBytesThreshold_ = 0;
-        size_t safePointsCounter_ = 0;
-        size_t safePointsCounterThreshold_ = 0;
-    };
+private:
+    void OnSafePointSlowPath() noexcept {
+        onSafePoint_(*this);
+        ClearCountersAndUpdateThresholds();
+    }
 
-    class GCData {
-    public:
-        using CurrentTimeCallback = std::function<uint64_t()>;
+    void ClearCountersAndUpdateThresholds() noexcept {
+        allocatedBytes_ = 0;
+        safePointsCounter_ = 0;
 
-        GCData(GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept;
+        allocatedBytesThreshold_ = config_.allocationThresholdBytes;
+        safePointsCounterThreshold_ = config_.threshold;
+    }
 
-        // May be called by different threads via `ThreadData`.
-        bool OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept;
+    GCSchedulerConfig& config_;
+    std::function<void(GCSchedulerThreadData&)> onSafePoint_;
 
-        // Always called by the GC thread.
-        void OnPerformFullGC() noexcept;
+    size_t allocatedBytes_ = 0;
+    size_t allocatedBytesThreshold_ = 0;
+    size_t safePointsCounter_ = 0;
+    size_t safePointsCounterThreshold_ = 0;
+};
 
-    private:
-        GCSchedulerConfig& config_;
-        CurrentTimeCallback currentTimeCallbackNs_;
+namespace internal {
 
-        std::atomic<uint64_t> timeOfLastGcNs_;
-    };
+KStdUniquePtr<GCSchedulerData> MakeEmptyGCSchedulerData() noexcept;
+KStdUniquePtr<GCSchedulerData> MakeGCSchedulerDataWithTimer(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept;
+KStdUniquePtr<GCSchedulerData> MakeGCSchedulerDataWithoutTimer(
+        GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept;
+KStdUniquePtr<GCSchedulerData> MakeGCSchedulerData(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept;
 
-    GCScheduler() noexcept;
+} // namespace internal
+
+class GCScheduler : private Pinned {
+public:
+    GCScheduler() noexcept = default;
 
     GCSchedulerConfig& config() noexcept { return config_; }
-    GCData& gcData() noexcept { return gcData_; }
-    ThreadData NewThreadData() noexcept;
+    // Only valid after `SetScheduleGC` is called.
+    GCSchedulerData& gcData() noexcept {
+        RuntimeAssert(gcData_ != nullptr, "Cannot be called before SetScheduleGC");
+        return *gcData_;
+    }
+
+    // Can only be called once.
+    void SetScheduleGC(std::function<void()> scheduleGC) noexcept;
+
+    GCSchedulerThreadData NewThreadData() noexcept {
+        return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->OnSafePoint(threadData); });
+    }
+
+    template <typename F>
+    KStdUniquePtr<GCSchedulerData> ReplaceGCSchedulerDataForTests(F&& factory) noexcept {
+        RuntimeAssert(static_cast<bool>(scheduleGC_), "Can only be called after SetScheduleGC");
+
+        auto other = std::forward<F>(factory)(config_, scheduleGC_);
+        RuntimeAssert(other != nullptr, "factory cannot return a null");
+        using std::swap;
+        swap(gcData_, other);
+        return other;
+    }
 
 private:
     GCSchedulerConfig config_;
-    GCData gcData_;
+    KStdUniquePtr<GCSchedulerData> gcData_;
+    std::function<void()> scheduleGC_;
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -51,52 +51,38 @@ struct FinalizeTraits {
     using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
 };
 
+// Global, because it's accessed on a hot path: avoid memory load from `this`.
+std::atomic<gc::SameThreadMarkAndSweep::SafepointFlag> gSafepointFlag = gc::SameThreadMarkAndSweep::SafepointFlag::kNone;
+
 } // namespace
 
-void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept {
-    SafePointRegular(GCScheduler::ThreadData::kFunctionEpilogueWeight);
+ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept {
+    SafePointRegular(GCSchedulerThreadData::kFunctionEpilogueWeight);
 }
 
-void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
-    SafePointRegular(GCScheduler::ThreadData::kLoopBodyWeight);
+ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
+    SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight);
 }
 
-void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept {
-    SafePointRegular(GCScheduler::ThreadData::kExceptionUnwindWeight);
+ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept {
+    SafePointRegular(GCSchedulerThreadData::kExceptionUnwindWeight);
 }
 
 void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
-    threadData_.suspensionData().suspendIfRequested();
-    auto& scheduler = threadData_.gcScheduler();
-    if (scheduler.OnSafePointAllocation(size)) {
-        RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointAllocation size=%zu", size);
-        PerformFullGC();
+    threadData_.gcScheduler().OnSafePointAllocation(size);
+    SafepointFlag flag = gSafepointFlag.load();
+    if (flag != SafepointFlag::kNone) {
+        SafePointSlowPath(flag);
     }
 }
 
 void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept {
-    mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
-    {
-        // Switch state to native to simulate this thread being a GC thread.
-        // As a bonus, if we failed to suspend threads (which means some other thread asked for a GC),
-        // we will automatically suspend at the scope exit.
-        // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`.
-        ThreadStateGuard guard(ThreadState::kNative);
-        finalizerQueue = gc_.PerformFullGC();
+    auto didGC = gc_.PerformFullGC();
+
+    if (!didGC) {
+        // If we failed to suspend threads, someone else might be asking to suspend them.
+        threadData_.suspensionData().suspendIfRequested();
     }
-
-    // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
-    // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing
-    // kotlin code.
-
-    // TODO: These will actually need to be run on a separate thread.
-    // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`.
-    AssertThreadState(ThreadState::kRunnable);
-    RuntimeLogDebug({kTagGC}, "Starting to run finalizers");
-    auto timeBeforeUs = konan::getTimeMicros();
-    finalizerQueue.Finalize();
-    auto timeAfterUs = konan::getTimeMicros();
-    RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs);
 }
 
 void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
@@ -104,102 +90,144 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
     PerformFullGC();
 }
 
-void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
+ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
+    threadData_.gcScheduler().OnSafePointRegular(weight);
+    SafepointFlag flag = gSafepointFlag.load();
+    if (flag != SafepointFlag::kNone) {
+        SafePointSlowPath(flag);
+    }
+}
+
+NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept {
+    RuntimeAssert(flag != SafepointFlag::kNone, "Must've been handled by the caller");
+    // No need to check for kNeedsSuspend, because `suspendIfRequested` checks for its own atomic.
     threadData_.suspensionData().suspendIfRequested();
-    auto& scheduler = threadData_.gcScheduler();
-    if (scheduler.OnSafePointRegular(weight)) {
-        RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointRegular weight=%zu", weight);
+    if (flag == SafepointFlag::kNeedsGC) {
+        RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint");
         PerformFullGC();
     }
 }
 
-mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
+gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept {
+    mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() {
+        RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
+        gSafepointFlag = SafepointFlag::kNeedsGC;
+    });
+}
+
+bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
     RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
     auto timeStartUs = konan::getTimeMicros();
-    bool didSuspend = mm::SuspendThreads();
-    auto timeSuspendUs = konan::getTimeMicros();
+    bool didSuspend = mm::RequestThreadsSuspension();
     if (!didSuspend) {
-        RuntimeLogDebug({kTagGC}, "Failed to suspend threads");
+        RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId());
         // Somebody else suspended the threads, and so ran a GC.
         // TODO: This breaks if suspension is used by something apart from GC.
-        return {};
+        return false;
     }
-    RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
+    RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
+    gSafepointFlag = SafepointFlag::kNeedsSuspend;
 
-    auto& scheduler = mm::GlobalData::Instance().gcScheduler();
-    scheduler.gcData().OnPerformFullGC();
+    mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
+    {
+        // Switch state to native to simulate this thread being a GC thread.
+        ThreadStateGuard guard(ThreadState::kNative);
 
-    RuntimeLogInfo({kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
-    KStdVector<ObjHeader*> graySet;
-    for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
-        // TODO: Maybe it's more efficient to do by the suspending thread?
-        thread.Publish();
-        thread.gcScheduler().OnStoppedForGC();
-        size_t stack = 0;
-        size_t tls = 0;
-        for (auto value : mm::ThreadRootSet(thread)) {
+        mm::WaitForThreadsSuspension();
+        auto timeSuspendUs = konan::getTimeMicros();
+        RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
+
+        auto& scheduler = mm::GlobalData::Instance().gcScheduler();
+        scheduler.gcData().OnPerformFullGC();
+
+        RuntimeLogInfo(
+                {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
+        KStdVector<ObjHeader*> graySet;
+        for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
+            // TODO: Maybe it's more efficient to do by the suspending thread?
+            thread.Publish();
+            thread.gcScheduler().OnStoppedForGC();
+            size_t stack = 0;
+            size_t tls = 0;
+            for (auto value : mm::ThreadRootSet(thread)) {
+                if (!isNullOrMarker(value.object)) {
+                    graySet.push_back(value.object);
+                    switch (value.source) {
+                        case mm::ThreadRootSet::Source::kStack:
+                            ++stack;
+                            break;
+                        case mm::ThreadRootSet::Source::kTLS:
+                            ++tls;
+                            break;
+                    }
+                }
+            }
+            RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
+        }
+        mm::StableRefRegistry::Instance().ProcessDeletions();
+        size_t global = 0;
+        size_t stableRef = 0;
+        for (auto value : mm::GlobalRootSet()) {
             if (!isNullOrMarker(value.object)) {
                 graySet.push_back(value.object);
                 switch (value.source) {
-                    case mm::ThreadRootSet::Source::kStack:
-                        ++stack;
+                    case mm::GlobalRootSet::Source::kGlobal:
+                        ++global;
                         break;
-                    case mm::ThreadRootSet::Source::kTLS:
-                        ++tls;
+                    case mm::GlobalRootSet::Source::kStableRef:
+                        ++stableRef;
                         break;
                 }
             }
         }
-        RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
+        auto timeRootSetUs = konan::getTimeMicros();
+        RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
+
+        // Can be unsafe, because we've stopped the world.
+        auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+
+        RuntimeLogInfo(
+                {kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(),
+                stableRef, timeRootSetUs - timeSuspendUs);
+        gc::Mark<MarkTraits>(std::move(graySet));
+        auto timeMarkUs = konan::getTimeMicros();
+        RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs);
+        finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
+        auto timeSweepUs = konan::getTimeMicros();
+        RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs);
+
+        // Can be unsafe, because we've stopped the world.
+        auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+
+        gSafepointFlag = SafepointFlag::kNone;
+        mm::ResumeThreads();
+        auto timeResumeUs = konan::getTimeMicros();
+
+        RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs);
+
+        auto finalizersCount = finalizerQueue.size();
+        auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
+
+        RuntimeLogInfo(
+                {kTagGC},
+                "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64
+                " microseconds",
+                epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs);
+        ++epoch_;
+        lastGCTimestampUs_ = timeResumeUs;
     }
-    mm::StableRefRegistry::Instance().ProcessDeletions();
-    size_t global = 0;
-    size_t stableRef = 0;
-    for (auto value : mm::GlobalRootSet()) {
-        if (!isNullOrMarker(value.object)) {
-            graySet.push_back(value.object);
-            switch (value.source) {
-                case mm::GlobalRootSet::Source::kGlobal:
-                    ++global;
-                    break;
-                case mm::GlobalRootSet::Source::kStableRef:
-                    ++stableRef;
-                    break;
-            }
-        }
-    }
-    auto timeRootSetUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
 
-    // Can be unsafe, because we've stopped the world.
-    auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+    // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
+    // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing
+    // kotlin code.
 
-    RuntimeLogInfo({kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(), stableRef, timeRootSetUs - timeSuspendUs);
-    gc::Mark<MarkTraits>(std::move(graySet));
-    auto timeMarkUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs);
-    auto finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
-    auto timeSweepUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs);
+    // TODO: These will actually need to be run on a separate thread.
+    AssertThreadState(ThreadState::kRunnable);
+    RuntimeLogDebug({kTagGC}, "Starting to run finalizers");
+    auto timeBeforeUs = konan::getTimeMicros();
+    finalizerQueue.Finalize();
+    auto timeAfterUs = konan::getTimeMicros();
+    RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs);
 
-    // Can be unsafe, because we've stopped the world.
-    auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
-
-    mm::ResumeThreads();
-    auto timeResumeUs = konan::getTimeMicros();
-
-    RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs);
-
-    auto finalizersCount = finalizerQueue.size();
-    auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
-
-    RuntimeLogInfo(
-            {kTagGC},
-            "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64
-            " microseconds",
-            epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs);
-    ++epoch_;
-    lastGCTimestampUs_ = timeResumeUs;
-
-    return finalizerQueue;
+    return true;
 }

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -24,6 +24,12 @@ namespace gc {
 // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads.
 class SameThreadMarkAndSweep : private Pinned {
 public:
+    enum class SafepointFlag {
+        kNone,
+        kNeedsSuspend,
+        kNeedsGC,
+    };
+
     class ObjectData {
     public:
         enum class Color {
@@ -57,16 +63,18 @@ public:
 
     private:
         void SafePointRegular(size_t weight) noexcept;
+        void SafePointSlowPath(SafepointFlag flag) noexcept;
 
         SameThreadMarkAndSweep& gc_;
         mm::ThreadData& threadData_;
     };
 
-    SameThreadMarkAndSweep() noexcept = default;
+    SameThreadMarkAndSweep() noexcept;
     ~SameThreadMarkAndSweep() = default;
 
 private:
-    mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueue PerformFullGC() noexcept;
+    // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
+    bool PerformFullGC() noexcept;
 
     size_t epoch_ = 0;
     uint64_t lastGCTimestampUs_ = 0;

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -215,9 +215,16 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader
 
 class SameThreadMarkAndSweepTest : public testing::Test {
 public:
+    SameThreadMarkAndSweepTest() {
+        mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests(
+                [](auto& config, auto scheduleGC) { return gc::internal::MakeEmptyGCSchedulerData(); });
+    }
+
     ~SameThreadMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().objectFactory().ClearForTests();
+        mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests(
+                [](auto& config, auto scheduleGC) { return gc::internal::MakeGCSchedulerData(config, std::move(scheduleGC)); });
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }

kotlin-native/runtime/src/main/cpp/SingleLockList.hpp

@@ -7,8 +7,10 @@
 #define RUNTIME_SINGLE_LOCK_LIST_H
 
 #include <cstddef>
+#include <iterator>
 #include <memory>
 #include <mutex>
+#include <type_traits>
 
 #include "Alloc.h"
 #include "Mutex.hpp"
@@ -52,6 +54,12 @@ public:
 
     class Iterator {
     public:
+        using difference_type = void;
+        using value_type = Value;
+        using pointer = Value*;
+        using reference = Value&;
+        using iterator_category = std::forward_iterator_tag;
+
         explicit Iterator(Node* node) noexcept : node_(node) {}
 
         Value& operator*() noexcept { return node_->value_; }

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -511,19 +511,19 @@ extern "C" void CheckGlobalsAccessible() {
     // Always accessible
 }
 
-extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() {
+extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointFunctionEpilogue() {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     AssertThreadState(threadData, ThreadState::kRunnable);
     threadData->gc().SafePointFunctionEpilogue();
 }
 
-extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() {
+extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointWhileLoopBody() {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     AssertThreadState(threadData, ThreadState::kRunnable);
     threadData->gc().SafePointLoopBody();
 }
 
-extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() {
+extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointExceptionUnwind() {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     AssertThreadState(threadData, ThreadState::kRunnable);
     threadData->gc().SafePointExceptionUnwind();

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -58,7 +58,7 @@ public:
 
     KStdVector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
 
-    gc::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; }
+    gc::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
 
     gc::GC::ThreadData& gc() noexcept { return gc_; }
 
@@ -83,7 +83,7 @@ private:
     ThreadLocalStorage tls_;
     StableRefRegistry::ThreadQueue stableRefThreadQueue_;
     ShadowStack shadowStack_;
-    gc::GCScheduler::ThreadData gcScheduler_;
+    gc::GCSchedulerThreadData gcScheduler_;
     gc::GC::ThreadData gc_;
     ObjectFactory<gc::GC>::ThreadQueue objectFactoryThreadQueue_;
     KStdVector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;

kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp

@@ -11,6 +11,7 @@
 #include <mutex>
 
 #include "Logging.hpp"
+#include "StackTrace.hpp"
 
 namespace {
 
@@ -60,7 +61,7 @@ NO_EXTERNAL_CALLS_CHECK void kotlin::mm::ThreadSuspensionData::suspendIfRequeste
     }
 }
 
-bool kotlin::mm::SuspendThreads() noexcept {
+NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::RequestThreadsSuspension() noexcept {
     RuntimeAssert(gSuspensionRequestedByCurrentThread == false, "Current thread already suspended threads.");
     {
         std::unique_lock lock(gSuspensionMutex);
@@ -72,12 +73,14 @@ bool kotlin::mm::SuspendThreads() noexcept {
     }
     gSuspensionRequestedByCurrentThread = true;
 
+    return true;
+}
+
+void kotlin::mm::WaitForThreadsSuspension() noexcept {
     // Spin wating for threads to suspend. Ignore Native threads.
     while(!allThreads(isSuspendedOrNative)) {
         yield();
     }
-
-    return true;
 }
 
 void kotlin::mm::ResumeThreads() noexcept {

kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp

@@ -54,13 +54,22 @@ private:
     void suspendIfRequestedSlowPath() noexcept;
 };
 
+bool RequestThreadsSuspension() noexcept;
+void WaitForThreadsSuspension() noexcept;
+
 /**
  * Suspends all threads registered in ThreadRegistry except threads that are in the Native state.
  * Blocks until all such threads are suspended. Threads that are in the Native state on the moment
  * of this call will be suspended on exit from the Native state.
  * Returns false if some other thread has suspended the threads.
  */
-bool SuspendThreads() noexcept;
+inline bool SuspendThreads() noexcept {
+    if (!RequestThreadsSuspension()) {
+        return false;
+    }
+    WaitForThreadsSuspension();
+    return true;
+}
 
 /**
  * Resumes all threads registered in ThreadRegistry that were suspended by the SuspendThreads call.