diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp new file mode 100644 index 00000000000..5546fad2a49 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -0,0 +1,203 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "SameThreadMarkAndSweep.hpp" + +#include + +#include "CompilerConstants.hpp" +#include "GlobalData.hpp" +#include "Logging.hpp" +#include "MarkAndSweepUtils.hpp" +#include "Memory.h" +#include "RootSet.hpp" +#include "Runtime.h" +#include "ThreadData.hpp" +#include "ThreadRegistry.hpp" +#include "ThreadSuspension.hpp" + +using namespace kotlin; + +namespace { + +struct MarkTraits { + static bool IsMarked(ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); + return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack; + } + + static bool TryMark(ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); + if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return false; + objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); + return true; + }; +}; + +struct SweepTraits { + using ObjectFactory = mm::ObjectFactory; + using ExtraObjectsFactory = mm::ExtraObjectDataFactory; + + static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { + auto *baseObject = object.GetBaseObject(); + if (!baseObject->heap()) return true; + auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).GCObjectData(); + return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack; + } + + static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { + auto& objectData = node.GCObjectData(); + if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite) return false; + objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite); + return true; + } +}; + +struct FinalizeTraits { + using ObjectFactory = mm::ObjectFactory; +}; + +// Global, because it's accessed on a hot path: avoid memory load from `this`. +std::atomic gSafepointFlag = gc::SameThreadMarkAndSweep::SafepointFlag::kNone; + +} // namespace + +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept { + SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight); +} + +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { + SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight); +} + +void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { + threadData_.gcScheduler().OnSafePointAllocation(size); + SafepointFlag flag = gSafepointFlag.load(); + if (flag != SafepointFlag::kNone) { + SafePointSlowPath(flag); + } +} + +void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept { + auto didGC = gc_.PerformFullGC(); + + if (!didGC) { + // If we failed to suspend threads, someone else might be asking to suspend them. + threadData_.suspensionData().suspendIfRequested(); + } +} + +void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { + RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); + PerformFullGC(); +} + +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { + threadData_.gcScheduler().OnSafePointRegular(weight); + SafepointFlag flag = gSafepointFlag.load(); + if (flag != SafepointFlag::kNone) { + SafePointSlowPath(flag); + } +} + +NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept { + RuntimeAssert(flag != SafepointFlag::kNone, "Must've been handled by the caller"); + // No need to check for kNeedsSuspend, because `suspendIfRequested` checks for its own atomic. + threadData_.suspensionData().suspendIfRequested(); + if (flag == SafepointFlag::kNeedsGC) { + RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint"); + PerformFullGC(); + } +} + +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept { + mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() { + RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); + gSafepointFlag = SafepointFlag::kNeedsGC; + }); +} + +bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { + RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); + auto timeStartUs = konan::getTimeMicros(); + bool didSuspend = mm::RequestThreadsSuspension(); + if (!didSuspend) { + RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId()); + // Somebody else suspended the threads, and so ran a GC. + // TODO: This breaks if suspension is used by something apart from GC. + return false; + } + RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId()); + gSafepointFlag = SafepointFlag::kNeedsSuspend; + + mm::ObjectFactory::FinalizerQueue finalizerQueue; + { + // Switch state to native to simulate this thread being a GC thread. + ThreadStateGuard guard(ThreadState::kNative); + + mm::WaitForThreadsSuspension(); + auto timeSuspendUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); + + auto& scheduler = mm::GlobalData::Instance().gcScheduler(); + scheduler.gcData().OnPerformFullGC(); + + RuntimeLogInfo( + {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); + auto graySet = collectRootSet(); + auto timeRootSetUs = konan::getTimeMicros(); + // Can be unsafe, because we've stopped the world. + auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + + RuntimeLogInfo( + {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(), + timeRootSetUs - timeSuspendUs); + auto markStats = gc::Mark(std::move(graySet)); + auto timeMarkUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries); + scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); + gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); + auto timeSweepExtraObjectsUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); + finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); + auto timeSweepUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs); + + // Can be unsafe, because we've stopped the world. + auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); + + gSafepointFlag = SafepointFlag::kNone; + mm::ResumeThreads(); + auto timeResumeUs = konan::getTimeMicros(); + + RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs); + + auto finalizersCount = finalizerQueue.size(); + auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount; + + RuntimeLogInfo( + {kTagGC}, + "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64 + " microseconds", + epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs); + ++epoch_; + lastGCTimestampUs_ = timeResumeUs; + } + + // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would + // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing + // kotlin code. + + // TODO: These will actually need to be run on a separate thread. + AssertThreadState(ThreadState::kRunnable); + RuntimeLogDebug({kTagGC}, "Starting to run finalizers"); + auto timeBeforeUs = konan::getTimeMicros(); + finalizerQueue.Finalize(); + auto timeAfterUs = konan::getTimeMicros(); + RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs); + + return true; +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp new file mode 100644 index 00000000000..5ee1b19edd8 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -0,0 +1,89 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include + +#include "Allocator.hpp" +#include "GCScheduler.hpp" +#include "ObjectFactory.hpp" +#include "Types.h" +#include "Utils.hpp" + +namespace kotlin { + +namespace mm { +class ThreadData; +} + +namespace gc { + +// Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads. +class SameThreadMarkAndSweep : private Pinned { +public: + enum class SafepointFlag { + kNone, + kNeedsSuspend, + kNeedsGC, + }; + + class ObjectData { + public: + enum class Color { + kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected. + // All new objects are allocated with this color. + kBlack, // Objects encountered during mark phase. + }; + + Color color() const noexcept { return color_; } + void setColor(Color color) noexcept { color_ = color; } + + private: + Color color_ = Color::kWhite; + }; + + class ThreadData : private Pinned { + public: + using ObjectData = SameThreadMarkAndSweep::ObjectData; + using Allocator = AllocatorWithGC; + + explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} + ~ThreadData() = default; + + void SafePointFunctionPrologue() noexcept; + void SafePointLoopBody() noexcept; + void SafePointExceptionUnwind() noexcept; + void SafePointAllocation(size_t size) noexcept; + + void PerformFullGC() noexcept; + + void OnOOM(size_t size) noexcept; + + Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } + + private: + void SafePointRegular(size_t weight) noexcept; + void SafePointSlowPath(SafepointFlag flag) noexcept; + + SameThreadMarkAndSweep& gc_; + mm::ThreadData& threadData_; + }; + + using Allocator = ThreadData::Allocator; + + SameThreadMarkAndSweep() noexcept; + ~SameThreadMarkAndSweep() = default; + +private: + // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). + bool PerformFullGC() noexcept; + + size_t epoch_ = 0; + uint64_t lastGCTimestampUs_ = 0; +}; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp new file mode 100644 index 00000000000..8a2e082bd98 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -0,0 +1,1165 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "SameThreadMarkAndSweep.hpp" + +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "ExtraObjectData.hpp" +#include "FinalizerHooksTestSupport.hpp" +#include "GlobalData.hpp" +#include "ObjectOps.hpp" +#include "ObjectTestSupport.hpp" +#include "TestSupport.hpp" +#include "ThreadData.hpp" + +using namespace kotlin; + +// These tests can only work if `GC` is `SameThreadMarkAndSweep`. +// TODO: Extracting GC into a separate module will help with this. + +namespace { + +struct Payload { + ObjHeader* field1; + ObjHeader* field2; + ObjHeader* field3; + + static constexpr std::array kFields = { + &Payload::field1, + &Payload::field2, + &Payload::field3, + }; +}; + +// TODO: This should go into test support for weak references. +struct WeakCounterPayload { + void* referred; + KInt lock; + KInt cookie; + + static constexpr std::array kFields{}; +}; + +using WeakCounter = test_support::Object; + +test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; +test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; +test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder()}; + +// TODO: Clean GlobalObjectHolder after it's gone. +class GlobalObjectHolder : private Pinned { +public: + explicit GlobalObjectHolder(mm::ThreadData& threadData) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); + mm::AllocateObject(&threadData, typeHolder.typeInfo(), &location_); + } + + GlobalObjectHolder(mm::ThreadData& threadData, ObjHeader* object) : location_(object) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); + } + + ObjHeader* header() { return location_; } + + test_support::Object& operator*() { return test_support::Object::FromObjHeader(location_); } + test_support::Object& operator->() { return test_support::Object::FromObjHeader(location_); } + +private: + ObjHeader* location_ = nullptr; +}; + +// TODO: Clean GlobalPermanentObjectHolder after it's gone. +class GlobalPermanentObjectHolder : private Pinned { +public: + explicit GlobalPermanentObjectHolder(mm::ThreadData& threadData) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &global_); + global_->typeInfoOrMeta_ = setPointerBits(global_->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); + RuntimeAssert(global_->permanent(), "Must be permanent"); + } + + ObjHeader* header() { return global_; } + + test_support::Object& operator*() { return object_; } + test_support::Object& operator->() { return object_; } + +private: + test_support::Object object_{typeHolder.typeInfo()}; + ObjHeader* global_{object_.header()}; +}; + +// TODO: Clean GlobalObjectArrayHolder after it's gone. +class GlobalObjectArrayHolder : private Pinned { +public: + explicit GlobalObjectArrayHolder(mm::ThreadData& threadData) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); + mm::AllocateArray(&threadData, theArrayTypeInfo, 3, &location_); + } + + ObjHeader* header() { return location_; } + + test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } + test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } + + ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } + +private: + ObjHeader* location_ = nullptr; +}; + +// TODO: Clean GlobalCharArrayHolder after it's gone. +class GlobalCharArrayHolder : private Pinned { +public: + explicit GlobalCharArrayHolder(mm::ThreadData& threadData) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); + mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, &location_); + } + + ObjHeader* header() { return location_; } + + test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } + test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } + +private: + ObjHeader* location_ = nullptr; +}; + +class StackObjectHolder : private Pinned { +public: + explicit StackObjectHolder(mm::ThreadData& threadData) { mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder_.slot()); } + explicit StackObjectHolder(test_support::Object& object) : holder_(object.header()) {} + explicit StackObjectHolder(ObjHeader* object) : holder_(object) {} + + ObjHeader* header() { return holder_.obj(); } + + test_support::Object& operator*() { return test_support::Object::FromObjHeader(holder_.obj()); } + test_support::Object& operator->() { return test_support::Object::FromObjHeader(holder_.obj()); } + +private: + ObjHolder holder_; +}; + +class StackObjectArrayHolder : private Pinned { +public: + explicit StackObjectArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theArrayTypeInfo, 3, holder_.slot()); } + + ObjHeader* header() { return holder_.obj(); } + + test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } + test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } + + ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } + +private: + ObjHolder holder_; +}; + +class StackCharArrayHolder : private Pinned { +public: + explicit StackCharArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, holder_.slot()); } + + ObjHeader* header() { return holder_.obj(); } + + test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } + test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } + +private: + ObjHolder holder_; +}; + +test_support::Object& AllocateObject(mm::ThreadData& threadData) { + ObjHolder holder; + mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder.slot()); + return test_support::Object::FromObjHeader(holder.obj()); +} + +test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threadData) { + ObjHolder holder; + mm::AllocateObject(&threadData, typeHolderWithFinalizer.typeInfo(), holder.slot()); + return test_support::Object::FromObjHeader(holder.obj()); +} + +KStdVector Alive(mm::ThreadData& threadData) { + KStdVector objects; + for (auto node : threadData.objectFactoryThreadQueue()) { + objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); + } + for (auto node : mm::GlobalData::Instance().objectFactory().LockForIter()) { + objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); + } + return objects; +} + +using Color = gc::SameThreadMarkAndSweep::ObjectData::Color; + +Color GetColor(ObjHeader* objHeader) { + auto nodeRef = mm::ObjectFactory::NodeRef::From(objHeader); + return nodeRef.GCObjectData().color(); +} + +WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { + mm::AllocateObject(&threadData, typeHolderWeakCounter.typeInfo(), location); + auto& weakCounter = WeakCounter::FromObjHeader(*location); + auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); + auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header()); + EXPECT_EQ(setCounter, weakCounter.header()); + weakCounter->referred = objHeader; + return weakCounter; +} + +class SameThreadMarkAndSweepTest : public testing::Test { +public: + + ~ConcurrentMarkAndSweepTest() { + mm::GlobalsRegistry::Instance().ClearForTests(); + mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); + mm::GlobalData::Instance().objectFactory().ClearForTests(); + } + + testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } + +private: + FinalizerHooksTestSupport finalizerHooks_; +}; + +} // namespace + +TEST_F(SameThreadMarkAndSweepTest, RootSet) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global1{threadData}; + GlobalObjectArrayHolder global2{threadData}; + GlobalCharArrayHolder global3{threadData}; + StackObjectHolder stack1{threadData}; + StackObjectArrayHolder stack2{threadData}; + StackCharArrayHolder stack3{threadData}; + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); + ASSERT_THAT(GetColor(global1.header()), Color::kWhite); + ASSERT_THAT(GetColor(global2.header()), Color::kWhite); + ASSERT_THAT(GetColor(global3.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); + EXPECT_THAT(GetColor(global1.header()), Color::kWhite); + EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(GetColor(global3.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global1{threadData}; + GlobalObjectArrayHolder global2{threadData}; + GlobalCharArrayHolder global3{threadData}; + StackObjectHolder stack1{threadData}; + StackObjectArrayHolder stack2{threadData}; + StackCharArrayHolder stack3{threadData}; + + global1->field1 = stack1.header(); + global1->field2 = global1.header(); + global1->field3 = global2.header(); + global2[0] = global1.header(); + global2[1] = global3.header(); + stack1->field1 = global1.header(); + stack1->field2 = stack1.header(); + stack1->field3 = stack2.header(); + stack2[0] = stack1.header(); + stack2[1] = stack3.header(); + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); + ASSERT_THAT(GetColor(global1.header()), Color::kWhite); + ASSERT_THAT(GetColor(global2.header()), Color::kWhite); + ASSERT_THAT(GetColor(global3.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); + EXPECT_THAT(GetColor(global1.header()), Color::kWhite); + EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(GetColor(global3.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, FreeObjects) { + RunInNewThread([](mm::ThreadData& threadData) { + auto& object1 = AllocateObject(threadData); + auto& object2 = AllocateObject(threadData); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { + RunInNewThread([this](mm::ThreadData& threadData) { + auto& object1 = AllocateObjectWithFinalizer(threadData); + auto& object2 = AllocateObjectWithFinalizer(threadData); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + + EXPECT_CALL(finalizerHook(), Call(object1.header())); + EXPECT_CALL(finalizerHook(), Call(object2.header())); + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { + RunInNewThread([](mm::ThreadData& threadData) { + auto& object1 = AllocateObject(threadData); + auto& weak1 = ([&threadData, &object1]() -> WeakCounter& { + ObjHolder holder; + return InstallWeakCounter(threadData, object1.header(), holder.slot()); + })(); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(weak1->referred, object1.header()); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { + RunInNewThread([](mm::ThreadData& threadData) { + auto& object1 = AllocateObject(threadData); + StackObjectHolder stack{threadData}; + auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(weak1->referred, object1.header()); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); + EXPECT_THAT(GetColor(weak1.header()), Color::kWhite); + EXPECT_THAT(weak1->referred, nullptr); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack{threadData}; + auto& object1 = AllocateObject(threadData); + auto& object2 = AllocateObject(threadData); + auto& object3 = AllocateObject(threadData); + auto& object4 = AllocateObject(threadData); + + global->field1 = object1.header(); + object1->field1 = object2.header(); + stack->field1 = object3.header(); + object3->field1 = object4.header(); + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); + ASSERT_THAT(GetColor(global.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(GetColor(object3.header()), Color::kWhite); + ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack.header()), Color::kWhite); + EXPECT_THAT(GetColor(object1.header()), Color::kWhite); + EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(GetColor(object3.header()), Color::kWhite); + EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack{threadData}; + auto& object1 = AllocateObject(threadData); + auto& object2 = AllocateObject(threadData); + auto& object3 = AllocateObject(threadData); + auto& object4 = AllocateObject(threadData); + auto& object5 = AllocateObject(threadData); + auto& object6 = AllocateObject(threadData); + + global->field1 = object1.header(); + object1->field1 = object2.header(); + object2->field1 = object1.header(); + stack->field1 = object3.header(); + object3->field1 = object4.header(); + object4->field1 = object3.header(); + object5->field1 = object6.header(); + object6->field1 = object5.header(); + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), + object5.header(), object6.header())); + ASSERT_THAT(GetColor(global.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(GetColor(object3.header()), Color::kWhite); + ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + ASSERT_THAT(GetColor(object5.header()), Color::kWhite); + ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack.header()), Color::kWhite); + EXPECT_THAT(GetColor(object1.header()), Color::kWhite); + EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(GetColor(object3.header()), Color::kWhite); + EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { + RunInNewThread([this](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack{threadData}; + auto& object1 = AllocateObjectWithFinalizer(threadData); + auto& object2 = AllocateObjectWithFinalizer(threadData); + auto& object3 = AllocateObjectWithFinalizer(threadData); + auto& object4 = AllocateObjectWithFinalizer(threadData); + auto& object5 = AllocateObjectWithFinalizer(threadData); + auto& object6 = AllocateObjectWithFinalizer(threadData); + + global->field1 = object1.header(); + object1->field1 = object2.header(); + object2->field1 = object1.header(); + stack->field1 = object3.header(); + object3->field1 = object4.header(); + object4->field1 = object3.header(); + object5->field1 = object6.header(); + object6->field1 = object5.header(); + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), + object5.header(), object6.header())); + ASSERT_THAT(GetColor(global.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(GetColor(object3.header()), Color::kWhite); + ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + ASSERT_THAT(GetColor(object5.header()), Color::kWhite); + ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + + EXPECT_CALL(finalizerHook(), Call(object5.header())); + EXPECT_CALL(finalizerHook(), Call(object6.header())); + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack.header()), Color::kWhite); + EXPECT_THAT(GetColor(object1.header()), Color::kWhite); + EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(GetColor(object3.header()), Color::kWhite); + EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack{threadData}; + auto& object1 = AllocateObject(threadData); + auto& object2 = AllocateObject(threadData); + + global->field1 = object1.header(); + object1->field1 = global.header(); + stack->field1 = object2.header(); + object2->field1 = stack.header(); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); + ASSERT_THAT(GetColor(global.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack.header()), Color::kWhite); + EXPECT_THAT(GetColor(object1.header()), Color::kWhite); + EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack{threadData}; + auto& object1 = AllocateObject(threadData); + auto& object2 = AllocateObject(threadData); + auto& object3 = AllocateObject(threadData); + auto& object4 = AllocateObject(threadData); + auto& object5 = AllocateObject(threadData); + auto& object6 = AllocateObject(threadData); + + global->field1 = object1.header(); + object1->field1 = object2.header(); + object2->field1 = object1.header(); + stack->field1 = object3.header(); + object3->field1 = object4.header(); + object4->field1 = object3.header(); + object5->field1 = object6.header(); + object6->field1 = object5.header(); + + ASSERT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), + object5.header(), object6.header())); + ASSERT_THAT(GetColor(global.header()), Color::kWhite); + ASSERT_THAT(GetColor(stack.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(GetColor(object3.header()), Color::kWhite); + ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + ASSERT_THAT(GetColor(object5.header()), Color::kWhite); + ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + threadData.gc().PerformFullGC(); + + EXPECT_THAT( + Alive(threadData), + testing::UnorderedElementsAre( + global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(stack.header()), Color::kWhite); + EXPECT_THAT(GetColor(object1.header()), Color::kWhite); + EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(GetColor(object3.header()), Color::kWhite); + EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalPermanentObjectHolder global1{threadData}; + GlobalObjectHolder global2{threadData}; + test_support::Object permanentObject{typeHolder.typeInfo()}; + permanentObject.header()->typeInfoOrMeta_ = + setPointerBits(permanentObject.header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); + RuntimeAssert(permanentObject.header()->permanent(), "Must be permanent"); + + global1->field1 = permanentObject.header(); + global2->field1 = global1.header(); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); + EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); + EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + }); +} + +TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) { + RunInNewThread([](mm::ThreadData& threadData) { + GlobalObjectHolder global{threadData}; + StackObjectHolder stack(*global); + auto& object = AllocateObject(threadData); + + global->field1 = object.header(); + + ASSERT_THAT(global.header(), stack.header()); + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(object.header()), Color::kWhite); + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); + EXPECT_THAT(GetColor(global.header()), Color::kWhite); + EXPECT_THAT(GetColor(object.header()), Color::kWhite); + }); +} + +namespace { + +class Mutator : private Pinned { +public: + Mutator() : thread_(&Mutator::RunLoop, this) {} + + ~Mutator() { + { + std::unique_lock guard(queueMutex_); + shutdownRequested_ = true; + } + queueCV_.notify_one(); + thread_.join(); + RuntimeAssert(queue_.empty(), "The queue must be empty, has size=%zu", queue_.size()); + RuntimeAssert(memory_ == nullptr, "Memory must have been deinitialized"); + RuntimeAssert(stackRoots_.empty(), "Stack roots must be empty, has size=%zu", stackRoots_.size()); + RuntimeAssert(globalRoots_.empty(), "Global roots must be empty, has size=%zu", globalRoots_.size()); + } + + template + [[nodiscard]] std::future Execute(F&& f) { + std::packaged_task task([this, f = std::forward(f)]() { f(*memory_->memoryState()->GetThreadData(), *this); }); + auto future = task.get_future(); + { + std::unique_lock guard(queueMutex_); + queue_.push_back(std::move(task)); + } + queueCV_.notify_one(); + return future; + } + + StackObjectHolder& AddStackRoot() { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData()); + auto& holderRef = *holder; + stackRoots_.push_back(std::move(holder)); + return holderRef; + } + + StackObjectHolder& AddStackRoot(ObjHeader* object) { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread"); + auto holder = make_unique(object); + auto& holderRef = *holder; + stackRoots_.push_back(std::move(holder)); + return holderRef; + } + + GlobalObjectHolder& AddGlobalRoot() { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData()); + auto& holderRef = *holder; + globalRoots_.push_back(std::move(holder)); + return holderRef; + } + + GlobalObjectHolder& AddGlobalRoot(ObjHeader* object) { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData(), object); + auto& holderRef = *holder; + globalRoots_.push_back(std::move(holder)); + return holderRef; + } + + KStdVector Alive() { return ::Alive(*memory_->memoryState()->GetThreadData()); } + +private: + void RunLoop() { + memory_ = make_unique(); + AssertThreadState(memory_->memoryState(), ThreadState::kRunnable); + + while (true) { + std::packaged_task task; + { + std::unique_lock guard(queueMutex_); + queueCV_.wait(guard, [this]() { return !queue_.empty() || shutdownRequested_; }); + if (shutdownRequested_) { + globalRoots_.clear(); + stackRoots_.clear(); + memory_.reset(); + return; + } + task = std::move(queue_.front()); + queue_.pop_front(); + } + task(); + } + } + + KStdUniquePtr memory_; + + // TODO: Consider full runtime init instead, and interact with initialized worker + std::condition_variable queueCV_; + std::mutex queueMutex_; + KStdDeque> queue_; + bool shutdownRequested_ = false; + std::thread thread_; + + KStdVector> globalRoots_; + KStdVector> stackRoots_; +}; + +} // namespace + +TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + local->field1 = reachable.header(); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAllCollect) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + local->field1 = reachable.header(); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + // TODO: Maybe check that only one GC is performed. + for (int i = 0; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto allocateInHeap = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = AllocateObject(threadData); + auto& local = AllocateObject(threadData); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + + local->field1 = reachable.header(); + + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + auto expandRootSet = [&globals, &locals](mm::ThreadData& threadData, Mutator& mutator, int i) { + mutator.AddGlobalRoot(globals[i]); + mutator.AddStackRoot(locals[i]); + }; + + mutators[0] + .Execute([expandRootSet, allocateInHeap](mm::ThreadData& threadData, Mutator& mutator) { + allocateInHeap(threadData, mutator, 0); + expandRootSet(threadData, mutator, 0); + }) + .wait(); + + // Allocate everything in heap before scheduling the GC. + for (int i = 1; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([allocateInHeap, i](mm::ThreadData& threadData, Mutator& mutator) { allocateInHeap(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { + expandRootSet(threadData, mutator, i); + threadData.gc().SafePointFunctionPrologue(); + }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(2 * kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable1 = AllocateObject(threadData); + auto& reachable2 = AllocateObject(threadData); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[2 * i] = reachable1.header(); + reachables[2 * i + 1] = reachable2.header(); + + // Expected to be run consequtively, so `reachables` for `j < i` are set. + if (i != 0) { + global->field1 = reachables[2 * (i - 1)]; + local->field1 = reachables[2 * (i - 1) + 1]; + } + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + // `expandRootSet` is expected to be run consequtively for each thread, so `.wait()` is required below. + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + // The last two are in fact unreachable. Their absence allows us to check that GC was in fact performed. + reachables.pop_back(); + reachables.pop_back(); + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { + KStdVector mutators(kDefaultThreadCount); + ObjHeader* globalRoot = nullptr; + WeakCounter* weak = nullptr; + + mutators[0] + .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { + auto& global = mutator.AddGlobalRoot(); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> WeakCounter& { + ObjHolder holder; + return InstallWeakCounter(threadData, object.header(), holder.slot()); + })(); + global->field1 = objectWeak.header(); + weak = &objectWeak; + globalRoot = global.header(); + }) + .wait(); + + // Make sure all mutators are initialized. + for (int i = 1; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PerformFullGC(); + EXPECT_THAT((*weak)->referred, nullptr); + }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().SafePointFunctionPrologue(); + EXPECT_THAT((*weak)->referred, nullptr); + }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); + } +} + +TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(2 * kDefaultThreadCount); + KStdVector locals(2 * kDefaultThreadCount); + KStdVector reachables(2 * kDefaultThreadCount); + KStdVector unreachables(2 * kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables, &unreachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable = AllocateObject(threadData); + auto& unreachable = AllocateObject(threadData); + local->field1 = reachable.header(); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + unreachables[i] = unreachable.header(); + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + // Now start attaching new threads. + KStdVector newMutators(kDefaultThreadCount); + KStdVector> attachFutures(kDefaultThreadCount); + + for (int i = 0; i < kDefaultThreadCount; ++i) { + attachFutures[i] = newMutators[i].Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i + kDefaultThreadCount); }); + } + + // All the other threads are stopping at safe points. + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); + } + + // GC will be completed first + for (auto& future : gcFutures) { + future.wait(); + } + + // Only then will the new threads be allowed to attach. + for (auto& future : attachFutures) { + future.wait(); + } + + // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything). + + KStdVector expectedAlive; + for (int i = 0; i < kDefaultThreadCount; ++i) { + expectedAlive.push_back(globals[i]); + expectedAlive.push_back(locals[i]); + expectedAlive.push_back(reachables[i]); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } + + for (int i = 0; i < kDefaultThreadCount; ++i) { + KStdVector aliveForThisThread(expectedAlive.begin(), expectedAlive.end()); + aliveForThisThread.push_back(globals[kDefaultThreadCount + i]); + aliveForThisThread.push_back(locals[kDefaultThreadCount + i]); + aliveForThisThread.push_back(reachables[kDefaultThreadCount + i]); + // Unreachables for new threads were not collected. + aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]); + EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread)); + } +} + + +TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { + KStdVector mutators(2); + std::atomic*> object1 = nullptr; + std::atomic weak = nullptr; + std::atomic done = false; + auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { + GlobalObjectHolder global1{threadData}; + auto& object1_local = AllocateObject(threadData); + object1 = &object1_local; + global1->field1 = object1_local.header(); + while (weak.load() == nullptr); + threadData.gc().PerformFullGC(); + + ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); + ASSERT_THAT(GetColor(global1.header()), Color::kWhite); + ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite); + ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite); + ASSERT_THAT((*weak.load())->referred, object1_local.header()); + + global1->field1 = nullptr; + + threadData.gc().PerformFullGC(); + + EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); + done = true; + }); + + auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { + while (object1.load() == nullptr) {} + ObjHolder holder; + auto &weak_local = InstallWeakCounter(threadData, object1.load()->header(), holder.slot()); + weak = &weak_local; + *holder.slot() = nullptr; + while (!done) threadData.gc().SafePointLoopBody(); + }); + + f0.wait(); + f1.wait(); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GC.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GC.hpp new file mode 100644 index 00000000000..b386cdb4808 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/GC.hpp @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "SameThreadMarkAndSweep.hpp" + +namespace kotlin { +namespace gc { + +using GC = kotlin::gc::SameThreadMarkAndSweep; + +inline constexpr bool kSupportsMultipleMutators = true; + +} // namespace gc +} // namespace kotlin +