Make SingleThreadMarkAndSweep support multiple threads

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp

@@ -11,6 +11,11 @@
 #include "Utils.hpp"
 
 namespace kotlin {
+
+namespace mm {
+class ThreadData;
+}
+
 namespace gc {
 
 // No-op GC is a GC that does not free memory.
@@ -23,7 +28,7 @@ public:
     public:
         using ObjectData = NoOpGC::ObjectData;
 
-        explicit ThreadData(NoOpGC& gc) noexcept {}
+        explicit ThreadData(NoOpGC& gc, mm::ThreadData& threadData) noexcept {}
         ~ThreadData() = default;
 
         void SafePointFunctionEpilogue() noexcept {}

kotlin-native/runtime/src/gc/stms/cpp/GC.hpp

@@ -13,7 +13,7 @@ namespace gc {
 
 using GC = kotlin::gc::SingleThreadMarkAndSweep;
 
-inline constexpr bool kSupportsMultipleMutators = false;
+inline constexpr bool kSupportsMultipleMutators = true;
 
 } // namespace gc
 } // namespace kotlin

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp

@@ -12,6 +12,7 @@
 #include "Runtime.h"
 #include "ThreadData.hpp"
 #include "ThreadRegistry.hpp"
+#include "ThreadSuspension.hpp"
 
 using namespace kotlin;
 
@@ -49,49 +50,76 @@ struct FinalizeTraits {
 } // namespace
 
 void gc::SingleThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept {
-    if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
-        PerformFullGC();
-    }
-    ++safePointsCounter_;
+    SafePointRegular(1);
 }
 
 void gc::SingleThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
-    if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
-        PerformFullGC();
-    }
-    ++safePointsCounter_;
+    SafePointRegular(1);
 }
 
 void gc::SingleThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept {
-    if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
-        PerformFullGC();
-    }
-    ++safePointsCounter_;
+    SafePointRegular(1);
 }
 
 void gc::SingleThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
     size_t allocationOverhead =
             gc_.GetAllocationThresholdBytes() == 0 ? allocatedBytes_ : allocatedBytes_ % gc_.GetAllocationThresholdBytes();
-    if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) {
+    if (threadData_.suspensionData().suspendIfRequested()) {
+        allocatedBytes_ = 0;
+    } else if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) {
+        allocatedBytes_ = 0;
         PerformFullGC();
     }
     allocatedBytes_ += size;
 }
 
 void gc::SingleThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept {
-    gc_.PerformFullGC();
+    mm::ObjectFactory<gc::SingleThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
+    {
+        // Switch state to native to simulate this thread being a GC thread.
+        // As a bonus, if we failed to suspend threads (which means some other thread asked for a GC),
+        // we will automatically suspend at the scope exit.
+        // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`.
+        ThreadStateGuard guard(ThreadState::kNative);
+        finalizerQueue = gc_.PerformFullGC();
+    }
+
+    // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
+    // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing
+    // kotlin code.
+
+    // TODO: These will actually need to be run on a separate thread.
+    // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`.
+    AssertThreadState(ThreadState::kRunnable);
+    finalizerQueue.Finalize();
 }
 
 void gc::SingleThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
     PerformFullGC();
 }
 
-void gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
-    RuntimeAssert(running_ == false, "Cannot have been called during another collection");
-    running_ = true;
+void gc::SingleThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
+    size_t counterOverhead = gc_.GetThreshold() == 0 ? safePointsCounter_ : safePointsCounter_ % gc_.GetThreshold();
+    if (threadData_.suspensionData().suspendIfRequested()) {
+        safePointsCounter_ = 0;
+    } else if (counterOverhead + weight >= gc_.GetThreshold()) {
+        safePointsCounter_ = 0;
+        PerformFullGC();
+    }
+    safePointsCounter_ += weight;
+}
+
+mm::ObjectFactory<gc::SingleThreadMarkAndSweep>::FinalizerQueue gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
+    bool didSuspend = mm::SuspendThreads();
+    if (!didSuspend) {
+        // Somebody else suspended the threads, and so ran a GC.
+        // TODO: This breaks if suspension is used by something apart from GC.
+        return {};
+    }
 
     KStdVector<ObjHeader*> graySet;
     for (auto& thread : mm::GlobalData::Instance().threadRegistry().Iter()) {
+        // TODO: Maybe it's more efficient to do by the suspending thread?
         thread.Publish();
         for (auto* object : mm::ThreadRootSet(thread)) {
             if (!isNullOrMarker(object)) {
@@ -109,10 +137,7 @@ void gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
     gc::Mark<MarkTraits>(std::move(graySet));
     auto finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
 
-    running_ = false;
+    mm::ResumeThreads();
 
-    // TODO: These will actually need to be run on a separate thread.
-    // TODO: This probably should check for the existence of runtime itself, but unit tests initialize only memory.
-    RuntimeAssert(mm::ThreadRegistry::Instance().CurrentThreadData() != nullptr, "Finalizers need a Kotlin runtime");
-    finalizerQueue.Finalize();
+    return finalizerQueue;
 }

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp

@@ -8,13 +8,20 @@
 
 #include <cstddef>
 
+#include "ObjectFactory.hpp"
 #include "Types.h"
 #include "Utils.hpp"
 
 namespace kotlin {
+
+namespace mm {
+class ThreadData;
+}
+
 namespace gc {
 
-// Stop-the-world Mark-and-Sweep for a single mutator
+// Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads.
+// TODO: Rename it away from SingleThreadMarkAndSweep, but keep it STMS.
 class SingleThreadMarkAndSweep : private Pinned {
 public:
     class ObjectData {
@@ -36,7 +43,7 @@ public:
     public:
         using ObjectData = SingleThreadMarkAndSweep::ObjectData;
 
-        explicit ThreadData(SingleThreadMarkAndSweep& gc) noexcept : gc_(gc) {}
+        explicit ThreadData(SingleThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
         ~ThreadData() = default;
 
         void SafePointFunctionEpilogue() noexcept;
@@ -49,7 +56,10 @@ public:
         void OnOOM(size_t size) noexcept;
 
     private:
+        void SafePointRegular(size_t weight) noexcept;
+
         SingleThreadMarkAndSweep& gc_;
+        mm::ThreadData& threadData_;
         size_t allocatedBytes_ = 0;
         size_t safePointsCounter_ = 0;
     };
@@ -67,9 +77,7 @@ public:
     bool GetAutoTune() noexcept { return autoTune_; }
 
 private:
-    void PerformFullGC() noexcept;
-
-    bool running_ = false;
+    mm::ObjectFactory<SingleThreadMarkAndSweep>::FinalizerQueue PerformFullGC() noexcept;
 
     size_t threshold_ = 1000;
     size_t allocationThresholdBytes_ = 10000;

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp

@@ -5,6 +5,11 @@
 
 #include "SingleThreadMarkAndSweep.hpp"
 
+#include <condition_variable>
+#include <future>
+#include <mutex>
+#include <thread>
+
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -58,6 +63,10 @@ public:
         mm::AllocateObject(&threadData, typeHolder.typeInfo(), &location_);
     }
 
+    GlobalObjectHolder(mm::ThreadData& threadData, ObjHeader* object) : location_(object) {
+        mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_);
+    }
+
     ObjHeader* header() { return location_; }
 
     test_support::Object<Payload>& operator*() { return test_support::Object<Payload>::FromObjHeader(location_); }
@@ -126,6 +135,7 @@ class StackObjectHolder : private Pinned {
 public:
     explicit StackObjectHolder(mm::ThreadData& threadData) { mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder_.slot()); }
     explicit StackObjectHolder(test_support::Object<Payload>& object) : holder_(object.header()) {}
+    explicit StackObjectHolder(ObjHeader* object) : holder_(object) {}
 
     ObjHeader* header() { return holder_.obj(); }
 
@@ -629,3 +639,404 @@ TEST_F(SingleThreadMarkAndSweepTest, SameObjectInRootSet) {
         EXPECT_THAT(GetColor(object.header()), Color::kWhite);
     });
 }
+
+namespace {
+
+class Mutator : private Pinned {
+public:
+    Mutator() : thread_(&Mutator::RunLoop, this) {}
+
+    ~Mutator() {
+        {
+            std::unique_lock guard(queueMutex_);
+            shutdownRequested_ = true;
+        }
+        queueCV_.notify_one();
+        thread_.join();
+        RuntimeAssert(queue_.empty(), "The queue must be empty, has size=%zu", queue_.size());
+        RuntimeAssert(memory_ == nullptr, "Memory must have been deinitialized");
+        RuntimeAssert(stackRoots_.empty(), "Stack roots must be empty, has size=%zu", stackRoots_.size());
+        RuntimeAssert(globalRoots_.empty(), "Global roots must be empty, has size=%zu", globalRoots_.size());
+    }
+
+    template <typename F>
+    [[nodiscard]] std::future<void> Execute(F&& f) {
+        std::packaged_task<void()> task([this, f = std::forward<F>(f)]() { f(*memory_->memoryState()->GetThreadData(), *this); });
+        auto future = task.get_future();
+        {
+            std::unique_lock guard(queueMutex_);
+            queue_.push_back(std::move(task));
+        }
+        queueCV_.notify_one();
+        return future;
+    }
+
+    StackObjectHolder& AddStackRoot() {
+        RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread");
+        auto holder = make_unique<StackObjectHolder>(*memory_->memoryState()->GetThreadData());
+        auto& holderRef = *holder;
+        stackRoots_.push_back(std::move(holder));
+        return holderRef;
+    }
+
+    StackObjectHolder& AddStackRoot(ObjHeader* object) {
+        RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread");
+        auto holder = make_unique<StackObjectHolder>(object);
+        auto& holderRef = *holder;
+        stackRoots_.push_back(std::move(holder));
+        return holderRef;
+    }
+
+    GlobalObjectHolder& AddGlobalRoot() {
+        RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread");
+        auto holder = make_unique<GlobalObjectHolder>(*memory_->memoryState()->GetThreadData());
+        auto& holderRef = *holder;
+        globalRoots_.push_back(std::move(holder));
+        return holderRef;
+    }
+
+    GlobalObjectHolder& AddGlobalRoot(ObjHeader* object) {
+        RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread");
+        auto holder = make_unique<GlobalObjectHolder>(*memory_->memoryState()->GetThreadData(), object);
+        auto& holderRef = *holder;
+        globalRoots_.push_back(std::move(holder));
+        return holderRef;
+    }
+
+    KStdVector<ObjHeader*> Alive() { return ::Alive(*memory_->memoryState()->GetThreadData()); }
+
+private:
+    void RunLoop() {
+        memory_ = make_unique<ScopedMemoryInit>();
+        AssertThreadState(memory_->memoryState(), ThreadState::kRunnable);
+
+        while (true) {
+            std::packaged_task<void()> task;
+            {
+                std::unique_lock guard(queueMutex_);
+                queueCV_.wait(guard, [this]() { return !queue_.empty() || shutdownRequested_; });
+                if (shutdownRequested_) {
+                    globalRoots_.clear();
+                    stackRoots_.clear();
+                    memory_.reset();
+                    return;
+                }
+                task = std::move(queue_.front());
+                queue_.pop_front();
+            }
+            task();
+        }
+    }
+
+    KStdUniquePtr<ScopedMemoryInit> memory_;
+
+    // TODO: Consider full runtime init instead, and interact with initialized worker
+    std::condition_variable queueCV_;
+    std::mutex queueMutex_;
+    KStdDeque<std::packaged_task<void()>> queue_;
+    bool shutdownRequested_ = false;
+    std::thread thread_;
+
+    KStdVector<KStdUniquePtr<GlobalObjectHolder>> globalRoots_;
+    KStdVector<KStdUniquePtr<StackObjectHolder>> stackRoots_;
+};
+
+} // namespace
+
+TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsCollect) {
+    KStdVector<Mutator> mutators(kDefaultThreadCount);
+    KStdVector<ObjHeader*> globals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> locals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> reachables(kDefaultThreadCount);
+
+    auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) {
+        auto& global = mutator.AddGlobalRoot();
+        auto& local = mutator.AddStackRoot();
+        auto& reachable = AllocateObject(threadData);
+        AllocateObject(threadData);
+        local->field1 = reachable.header();
+        globals[i] = global.header();
+        locals[i] = local.header();
+        reachables[i] = reachable.header();
+    };
+
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        mutators[i]
+                .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); })
+                .wait();
+    }
+
+    KStdVector<std::future<void>> gcFutures(kDefaultThreadCount);
+
+    gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); });
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        gcFutures[i] =
+                mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); });
+    }
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+
+    KStdVector<ObjHeader*> expectedAlive;
+    for (auto& global : globals) {
+        expectedAlive.push_back(global);
+    }
+    for (auto& local : locals) {
+        expectedAlive.push_back(local);
+    }
+    for (auto& reachable : reachables) {
+        expectedAlive.push_back(reachable);
+    }
+
+    for (auto& mutator : mutators) {
+        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
+    }
+}
+
+TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsAllCollect) {
+    KStdVector<Mutator> mutators(kDefaultThreadCount);
+    KStdVector<ObjHeader*> globals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> locals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> reachables(kDefaultThreadCount);
+
+    auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) {
+        auto& global = mutator.AddGlobalRoot();
+        auto& local = mutator.AddStackRoot();
+        auto& reachable = AllocateObject(threadData);
+        AllocateObject(threadData);
+        local->field1 = reachable.header();
+        globals[i] = global.header();
+        locals[i] = local.header();
+        reachables[i] = reachable.header();
+    };
+
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        mutators[i]
+                .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); })
+                .wait();
+    }
+
+    KStdVector<std::future<void>> gcFutures(kDefaultThreadCount);
+
+    // TODO: Maybe check that only one GC is performed.
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        gcFutures[i] = mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); });
+    }
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+
+    KStdVector<ObjHeader*> expectedAlive;
+    for (auto& global : globals) {
+        expectedAlive.push_back(global);
+    }
+    for (auto& local : locals) {
+        expectedAlive.push_back(local);
+    }
+    for (auto& reachable : reachables) {
+        expectedAlive.push_back(reachable);
+    }
+
+    for (auto& mutator : mutators) {
+        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
+    }
+}
+
+TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) {
+    KStdVector<Mutator> mutators(kDefaultThreadCount);
+    KStdVector<ObjHeader*> globals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> locals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> reachables(kDefaultThreadCount);
+
+    auto allocateInHeap = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) {
+        auto& global = AllocateObject(threadData);
+        auto& local = AllocateObject(threadData);
+        auto& reachable = AllocateObject(threadData);
+        AllocateObject(threadData);
+
+        local->field1 = reachable.header();
+
+        globals[i] = global.header();
+        locals[i] = local.header();
+        reachables[i] = reachable.header();
+    };
+
+    auto expandRootSet = [&globals, &locals](mm::ThreadData& threadData, Mutator& mutator, int i) {
+        mutator.AddGlobalRoot(globals[i]);
+        mutator.AddStackRoot(locals[i]);
+    };
+
+    mutators[0]
+            .Execute([expandRootSet, allocateInHeap](mm::ThreadData& threadData, Mutator& mutator) {
+                allocateInHeap(threadData, mutator, 0);
+                expandRootSet(threadData, mutator, 0);
+            })
+            .wait();
+
+    // Allocate everything in heap before scheduling the GC.
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        mutators[i]
+                .Execute([allocateInHeap, i](mm::ThreadData& threadData, Mutator& mutator) { allocateInHeap(threadData, mutator, i); })
+                .wait();
+    }
+
+    KStdVector<std::future<void>> gcFutures(kDefaultThreadCount);
+    gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); });
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        gcFutures[i] = mutators[i].Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) {
+            expandRootSet(threadData, mutator, i);
+            threadData.gc().SafePointFunctionEpilogue();
+        });
+    }
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+
+    KStdVector<ObjHeader*> expectedAlive;
+    for (auto& global : globals) {
+        expectedAlive.push_back(global);
+    }
+    for (auto& local : locals) {
+        expectedAlive.push_back(local);
+    }
+    for (auto& reachable : reachables) {
+        expectedAlive.push_back(reachable);
+    }
+
+    for (auto& mutator : mutators) {
+        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
+    }
+}
+
+TEST_F(SingleThreadMarkAndSweepTest, CrossThreadReference) {
+    KStdVector<Mutator> mutators(kDefaultThreadCount);
+    KStdVector<ObjHeader*> globals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> locals(kDefaultThreadCount);
+    KStdVector<ObjHeader*> reachables(2 * kDefaultThreadCount);
+
+    auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) {
+        auto& global = mutator.AddGlobalRoot();
+        auto& local = mutator.AddStackRoot();
+        auto& reachable1 = AllocateObject(threadData);
+        auto& reachable2 = AllocateObject(threadData);
+        globals[i] = global.header();
+        locals[i] = local.header();
+        reachables[2 * i] = reachable1.header();
+        reachables[2 * i + 1] = reachable2.header();
+
+        // Expected to be run consequtively, so `reachables` for `j < i` are set.
+        if (i != 0) {
+            global->field1 = reachables[2 * (i - 1)];
+            local->field1 = reachables[2 * (i - 1) + 1];
+        }
+    };
+
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        // `expandRootSet` is expected to be run consequtively for each thread, so `.wait()` is required below.
+        mutators[i]
+                .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); })
+                .wait();
+    }
+
+    KStdVector<std::future<void>> gcFutures(kDefaultThreadCount);
+
+    gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); });
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        gcFutures[i] =
+                mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); });
+    }
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+
+    KStdVector<ObjHeader*> expectedAlive;
+    for (auto& global : globals) {
+        expectedAlive.push_back(global);
+    }
+    for (auto& local : locals) {
+        expectedAlive.push_back(local);
+    }
+    // The last two are in fact unreachable. Their absence allows us to check that GC was in fact performed.
+    reachables.pop_back();
+    reachables.pop_back();
+    for (auto& reachable : reachables) {
+        expectedAlive.push_back(reachable);
+    }
+
+    for (auto& mutator : mutators) {
+        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
+    }
+}
+
+TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsWeaks) {
+    KStdVector<Mutator> mutators(kDefaultThreadCount);
+    ObjHeader* globalRoot = nullptr;
+    WeakCounter* weak = nullptr;
+
+    mutators[0]
+            .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) {
+                auto& global = mutator.AddGlobalRoot();
+
+                auto& object = AllocateObject(threadData);
+                auto& objectWeak = ([&threadData, &object]() -> WeakCounter& {
+                    ObjHolder holder;
+                    return InstallWeakCounter(threadData, object.header(), holder.slot());
+                })();
+                global->field1 = objectWeak.header();
+                weak = &objectWeak;
+                globalRoot = global.header();
+            })
+            .wait();
+
+    // Make sure all mutators are initialized.
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait();
+    }
+
+    KStdVector<std::future<void>> gcFutures(kDefaultThreadCount);
+
+    gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
+        threadData.gc().PerformFullGC();
+        EXPECT_THAT((*weak)->referred, nullptr);
+    });
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (int i = 1; i < kDefaultThreadCount; ++i) {
+        gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
+            threadData.gc().SafePointFunctionEpilogue();
+            EXPECT_THAT((*weak)->referred, nullptr);
+        });
+    }
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+
+    for (auto& mutator : mutators) {
+        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header()));
+    }
+}
+
+// TODO: Attaching new threads while GC is in progress.

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp

@@ -79,7 +79,10 @@ void mm::ExtraObjectData::ClearWeakReferenceCounter() noexcept {
     if (!HasWeakReferenceCounter()) return;
 
     WeakReferenceCounterClear(weakReferenceCounter_);
-    mm::SetHeapRef(&weakReferenceCounter_, nullptr);
+    // Not using `mm::SetHeapRef here`, because this code is called during sweep phase by the GC thread,
+    // and so cannot affect marking.
+    // TODO: Asserts on the above?
+    weakReferenceCounter_ = nullptr;
 }
 
 mm::ExtraObjectData::~ExtraObjectData() {

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -515,6 +515,7 @@ public:
             auto* heapObject = new (node.Data()) HeapObjHeader();
             auto* object = &heapObject->object;
             object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
+            // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
             return object;
         }
 
@@ -528,6 +529,7 @@ public:
             auto* array = &heapArray->array;
             array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
             array->count_ = count;
+            // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
             return array;
         }
 

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -33,7 +33,7 @@ public:
         threadId_(threadId),
         globalsThreadQueue_(GlobalsRegistry::Instance()),
         stableRefThreadQueue_(StableRefRegistry::Instance()),
-        gc_(GlobalData::Instance().gc()),
+        gc_(GlobalData::Instance().gc(), *this),
         objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_),
         suspensionData_(ThreadState::kRunnable) {}
 

kotlin-native/runtime/src/mm/cpp/ThreadState.cpp

@@ -42,4 +42,4 @@ ALWAYS_INLINE void kotlin::AssertThreadState(MemoryState* thread, std::initializ
 
 ThreadState kotlin::GetThreadState(MemoryState* thread) noexcept {
     return thread->GetThreadData()->state();
-}
+}

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -33,7 +33,8 @@ kotlin::test_support::TypeInfoHolder theObjCObjectWrapperTypeInfoHolder{
         kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
 kotlin::test_support::TypeInfoHolder theOpaqueFunctionTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
 kotlin::test_support::TypeInfoHolder theShortArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<KShort>()};
-kotlin::test_support::TypeInfoHolder theStringTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<KChar>()};
+kotlin::test_support::TypeInfoHolder theStringTypeInfoHolder{
+        kotlin::test_support::TypeInfoHolder::ArrayBuilder<KChar>().addFlag(TF_IMMUTABLE)};
 kotlin::test_support::TypeInfoHolder theThrowableTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
 kotlin::test_support::TypeInfoHolder theUnitTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
 kotlin::test_support::TypeInfoHolder theWorkerBoundReferenceTypeInfoHolder{