diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index f75f08a7098..65758d8367b 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -397,15 +397,15 @@ bitcode { testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc_custom", "objc") } - testsGroup("experimentalMM_mimalloc_runtime_tests") { + testsGroup("experimentalMM_stms_mimalloc_runtime_tests") { testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc") } - testsGroup("experimentalMM_std_alloc_runtime_tests") { + testsGroup("experimentalMM_stms_std_alloc_runtime_tests") { testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "legacy_alloc", "objc") } - testsGroup("experimentalMM_custom_alloc_runtime_tests") { + testsGroup("experimentalMM_stms_custom_alloc_runtime_tests") { testedModules.addAll("mm", "same_thread_ms_gc_custom") testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index c0e31d2814e..1da882953d4 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -1,223 +1,27 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ #include "ConcurrentMarkAndSweep.hpp" -#include -#include #include -#include -#include -#include "gmock/gmock.h" #include "gtest/gtest.h" -#include "AllocatorTestSupport.hpp" -#include "ExtraObjectData.hpp" -#include "FinalizerHooksTestSupport.hpp" #include "GCImpl.hpp" #include "GlobalData.hpp" -#include "ObjectOps.hpp" -#include "ObjectTestSupport.hpp" #include "SafePoint.hpp" -#include "SingleThreadExecutor.hpp" #include "StableRef.hpp" #include "TestSupport.hpp" -#include "ThreadData.hpp" -#include "WeakRef.hpp" +#include "TracingGCTest.hpp" using namespace kotlin; -// These tests can only work if `GC` is `ConcurrentMarkAndSweep`. - namespace { -struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; - - static constexpr std::array kFields = { - &Payload::field1, - &Payload::field2, - &Payload::field3, - }; -}; - -test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; -test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; - -// TODO: Clean GlobalObjectHolder after it's gone. -class GlobalObjectHolder : private Pinned { +class ConcurrentMarkAndSweepTest { public: - explicit GlobalObjectHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateObject(&threadData, typeHolder.typeInfo(), &location_); - } - - GlobalObjectHolder(mm::ThreadData& threadData, ObjHeader* object) : location_(object) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::Object& operator*() { return test_support::Object::FromObjHeader(location_); } - test_support::Object& operator->() { return test_support::Object::FromObjHeader(location_); } - -private: - ObjHeader* location_ = nullptr; -}; - -// TODO: Clean GlobalPermanentObjectHolder after it's gone. -class GlobalPermanentObjectHolder : private Pinned { -public: - explicit GlobalPermanentObjectHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &global_); - global_->typeInfoOrMeta_ = setPointerBits(global_->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); - RuntimeAssert(global_->permanent(), "Must be permanent"); - } - - ObjHeader* header() { return global_; } - - test_support::Object& operator*() { return object_; } - test_support::Object& operator->() { return object_; } - -private: - test_support::Object object_{typeHolder.typeInfo()}; - ObjHeader* global_{object_.header()}; -}; - -// TODO: Clean GlobalObjectArrayHolder after it's gone. -class GlobalObjectArrayHolder : private Pinned { -public: - explicit GlobalObjectArrayHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateArray(&threadData, theArrayTypeInfo, 3, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } - test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } - - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } - -private: - ObjHeader* location_ = nullptr; -}; - -// TODO: Clean GlobalCharArrayHolder after it's gone. -class GlobalCharArrayHolder : private Pinned { -public: - explicit GlobalCharArrayHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } - test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } - -private: - ObjHeader* location_ = nullptr; -}; - -class StackObjectHolder : private Pinned { -public: - explicit StackObjectHolder(mm::ThreadData& threadData) { mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder_.slot()); } - explicit StackObjectHolder(test_support::Object& object) : holder_(object.header()) {} - explicit StackObjectHolder(ObjHeader* object) : holder_(object) {} - - ObjHeader* header() { return holder_.obj(); } - - test_support::Object& operator*() { return test_support::Object::FromObjHeader(holder_.obj()); } - test_support::Object& operator->() { return test_support::Object::FromObjHeader(holder_.obj()); } - -private: - ObjHolder holder_; -}; - -class StackObjectArrayHolder : private Pinned { -public: - explicit StackObjectArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theArrayTypeInfo, 3, holder_.slot()); } - - ObjHeader* header() { return holder_.obj(); } - - test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } - test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } - - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } - -private: - ObjHolder holder_; -}; - -class StackCharArrayHolder : private Pinned { -public: - explicit StackCharArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, holder_.slot()); } - - ObjHeader* header() { return holder_.obj(); } - - test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } - test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } - -private: - ObjHolder holder_; -}; - -test_support::Object& AllocateObject(mm::ThreadData& threadData) { - ObjHolder holder; - mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder.slot()); - return test_support::Object::FromObjHeader(holder.obj()); -} - -test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threadData) { - ObjHolder holder; - mm::AllocateObject(&threadData, typeHolderWithFinalizer.typeInfo(), holder.slot()); - return test_support::Object::FromObjHeader(holder.obj()); -} - -std::vector Alive(mm::ThreadData& threadData) { - return alloc::test_support::allocatedObjects(threadData); -} - -test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { - mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); - auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); - auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); - weakReference->weakRef = static_cast(mm::WeakRef::create(objHeader)); - weakReference->referred = objHeader; - auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header()); - EXPECT_EQ(setWeakRef, weakReference.header()); - return weakReference; -} - -struct ParallelismOptions { - std::size_t maxParallelism; - bool cooperativeMutators; - std::size_t auxGCThreads; -}; - -class ConcurrentMarkAndSweepTest : public testing::TestWithParam { -public: - - ConcurrentMarkAndSweepTest() { - if (supportedConfiguration()) { - mm::GlobalData::Instance().gc().impl().gc().reconfigure(GetParam().maxParallelism, - GetParam().cooperativeMutators, - GetParam().auxGCThreads); - } - } - - void SetUp() override { - if (!supportedConfiguration()) { - GTEST_SKIP() << "Unsupported parallelism configuration"; - } - } - ~ConcurrentMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); @@ -225,754 +29,12 @@ public: mm::GlobalData::Instance().allocator().clearForTests(); } - testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } - -private: - bool supportedConfiguration() const { - return !compiler::gcMarkSingleThreaded() || (!GetParam().cooperativeMutators && GetParam().auxGCThreads == 0); - } - - FinalizerHooksTestSupport finalizerHooks_; + void SetUp() {} }; } // namespace -TEST_P(ConcurrentMarkAndSweepTest, RootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global1{threadData}; - GlobalObjectArrayHolder global2{threadData}; - GlobalCharArrayHolder global3{threadData}; - StackObjectHolder stack1{threadData}; - StackObjectArrayHolder stack2{threadData}; - StackCharArrayHolder stack3{threadData}; - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(global2.header()), false); - ASSERT_THAT(gc::isMarked(global3.header()), false); - ASSERT_THAT(gc::isMarked(stack1.header()), false); - ASSERT_THAT(gc::isMarked(stack2.header()), false); - ASSERT_THAT(gc::isMarked(stack3.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(gc::isMarked(global1.header()), false); - EXPECT_THAT(gc::isMarked(global2.header()), false); - EXPECT_THAT(gc::isMarked(global3.header()), false); - EXPECT_THAT(gc::isMarked(stack1.header()), false); - EXPECT_THAT(gc::isMarked(stack2.header()), false); - EXPECT_THAT(gc::isMarked(stack3.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global1{threadData}; - GlobalObjectArrayHolder global2{threadData}; - GlobalCharArrayHolder global3{threadData}; - StackObjectHolder stack1{threadData}; - StackObjectArrayHolder stack2{threadData}; - StackCharArrayHolder stack3{threadData}; - - global1->field1 = stack1.header(); - global1->field2 = global1.header(); - global1->field3 = global2.header(); - global2[0] = global1.header(); - global2[1] = global3.header(); - stack1->field1 = global1.header(); - stack1->field2 = stack1.header(); - stack1->field3 = stack2.header(); - stack2[0] = stack1.header(); - stack2[1] = stack3.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(global2.header()), false); - ASSERT_THAT(gc::isMarked(global3.header()), false); - ASSERT_THAT(gc::isMarked(stack1.header()), false); - ASSERT_THAT(gc::isMarked(stack2.header()), false); - ASSERT_THAT(gc::isMarked(stack3.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(gc::isMarked(global1.header()), false); - EXPECT_THAT(gc::isMarked(global2.header()), false); - EXPECT_THAT(gc::isMarked(global3.header()), false); - EXPECT_THAT(gc::isMarked(stack1.header()), false); - EXPECT_THAT(gc::isMarked(stack2.header()), false); - EXPECT_THAT(gc::isMarked(stack3.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) { - RunInNewThread([](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { - RunInNewThread([this](mm::ThreadData& threadData) { - auto& object1 = AllocateObjectWithFinalizer(threadData); - auto& object2 = AllocateObjectWithFinalizer(threadData); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - EXPECT_CALL(finalizerHook(), Call(object1.header())); - EXPECT_CALL(finalizerHook(), Call(object2.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { - RunInNewThread([this](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object1.header(), holder.slot()); - })(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(weak1.header()), false); - ASSERT_THAT(weak1.get(), object1.header()); - - EXPECT_CALL(finalizerHook(), Call(weak1.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) { - RunInNewThread([](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - StackObjectHolder stack{threadData}; - auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(weak1.header()), false); - ASSERT_THAT(weak1.get(), object1.header()); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(gc::isMarked(weak1.header()), false); - EXPECT_THAT(weak1.get(), nullptr); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - auto& object5 = AllocateObject(threadData); - auto& object6 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { - RunInNewThread([this](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObjectWithFinalizer(threadData); - auto& object2 = AllocateObjectWithFinalizer(threadData); - auto& object3 = AllocateObjectWithFinalizer(threadData); - auto& object4 = AllocateObjectWithFinalizer(threadData); - auto& object5 = AllocateObjectWithFinalizer(threadData); - auto& object6 = AllocateObjectWithFinalizer(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - EXPECT_CALL(finalizerHook(), Call(object5.header())); - EXPECT_CALL(finalizerHook(), Call(object6.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = global.header(); - stack->field1 = object2.header(); - object2->field1 = stack.header(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - auto& object5 = AllocateObject(threadData); - auto& object6 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalPermanentObjectHolder global1{threadData}; - GlobalObjectHolder global2{threadData}; - test_support::Object permanentObject{typeHolder.typeInfo()}; - permanentObject.header()->typeInfoOrMeta_ = - setPointerBits(permanentObject.header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); - RuntimeAssert(permanentObject.header()->permanent(), "Must be permanent"); - - global1->field1 = permanentObject.header(); - global2->field1 = global1.header(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(gc::isMarked(global2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(gc::isMarked(global2.header()), false); - }); -} - -TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack(*global); - auto& object = AllocateObject(threadData); - - global->field1 = object.header(); - - ASSERT_THAT(global.header(), stack.header()); - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(object.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(object.header()), false); - }); -} - -namespace { - -class Mutator : private Pinned { -public: - template - [[nodiscard]] std::future Execute(F&& f) { - return executor_.execute( - [this, f = std::forward(f)] { f(*executor_.context().memory_->memoryState()->GetThreadData(), *this); }); - } - - StackObjectHolder& AddStackRoot() { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddStackRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData()); - auto& holderRef = *holder; - context.stackRoots_.push_back(std::move(holder)); - return holderRef; - } - - StackObjectHolder& AddStackRoot(ObjHeader* object) { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddStackRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(object); - auto& holderRef = *holder; - context.stackRoots_.push_back(std::move(holder)); - return holderRef; - } - - GlobalObjectHolder& AddGlobalRoot() { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddGlobalRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData()); - auto& holderRef = *holder; - context.globalRoots_.push_back(std::move(holder)); - return holderRef; - } - - GlobalObjectHolder& AddGlobalRoot(ObjHeader* object) { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddGlobalRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData(), object); - auto& holderRef = *holder; - context.globalRoots_.push_back(std::move(holder)); - return holderRef; - } - - std::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } - -private: - struct Context { - std::unique_ptr memory_; - std::vector> stackRoots_; - std::vector> globalRoots_; - - Context() : memory_(std::make_unique()) { - // SingleThreadExecutor must work in the runnable state, so that GC does not collect between tasks. - AssertThreadState(memory_->memoryState(), ThreadState::kRunnable); - } - }; - - SingleThreadExecutor executor_; -}; - -} // namespace - -TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsCollect) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - std::atomic gcDone = false; - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsAllCollect) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - // If GC starts before all thread executed line above, two gc will be run - // So we temporary switch threads to native state and then return them back after all GC runs are done - SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative); - })); - } - - for (auto& future : gcFutures) { - future.wait(); - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([](mm::ThreadData& threadData, Mutator& mutator) { - SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kRunnable); - }) - .wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto allocateInHeap = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = AllocateObject(threadData); - auto& local = AllocateObject(threadData); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - - local->field1 = reachable.header(); - - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - auto expandRootSet = [&globals, &locals](mm::ThreadData& threadData, Mutator& mutator, int i) { - mutator.AddGlobalRoot(globals[i]); - mutator.AddStackRoot(locals[i]); - }; - - // Allocate everything in heap before scheduling the GC. - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([allocateInHeap, i](mm::ThreadData& threadData, Mutator& mutator) { allocateInHeap(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - gcFutures.emplace_back(mutators[i].Execute([&gcDone, i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { - expandRootSet(threadData, mutator, i); - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_P(ConcurrentMarkAndSweepTest, CrossThreadReference) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(2 * kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable1 = AllocateObject(threadData); - auto& reachable2 = AllocateObject(threadData); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[2 * i] = reachable1.header(); - reachables[2 * i + 1] = reachable2.header(); - - // Expected to be run consequtively, so `reachables` for `j < i` are set. - if (i != 0) { - global->field1 = reachables[2 * (i - 1)]; - local->field1 = reachables[2 * (i - 1) + 1]; - } - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - // `expandRootSet` is expected to be run consequtively for each thread, so `.wait()` is required below. - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - std::atomic gcDone = false; - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - // The last two are in fact unreachable. Their absence allows us to check that GC was in fact performed. - reachables.pop_back(); - reachables.pop_back(); - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { +TYPED_TEST_P(TracingGCTest, CMSMultipleMutatorsWeak) { std::vector mutators(kDefaultThreadCount); ObjHeader* globalRoot = nullptr; test_support::RegularWeakReferenceImpl* weak = nullptr; @@ -1002,15 +64,21 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { std::atomic gcDone = false; // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } + while (!mm::IsThreadSuspensionRequested()) {} for (auto& mutator : mutators) { gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { + bool dead = false; while (!gcDone.load(std::memory_order_relaxed)) { mm::safePoint(threadData); - EXPECT_THAT(weak->get(), nullptr); + auto weakReferee = weak->get(); + if (dead) { + EXPECT_THAT(weakReferee, nullptr); + } else if (weakReferee == nullptr) { + dead = true; + } } + EXPECT_THAT(weak->get(), nullptr); })); } @@ -1026,213 +94,5 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { } } -TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeakNewObj) { - std::vector mutators(kDefaultThreadCount); - - // Make sure all mutators are initialized. - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - mm::safePoint(threadData); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - EXPECT_NE(objectWeak.get(), nullptr); - - auto& extraObj = *mm::ExtraObjectData::Get(object.header()); - extraObj.ClearRegularWeakReferenceImpl(); - extraObj.Uninstall(); - alloc::destroyExtraObjectData(extraObj); - - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } -} - -TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(2 * kDefaultThreadCount); - std::vector locals(2 * kDefaultThreadCount); - std::vector reachables(2 * kDefaultThreadCount); - std::vector unreachables(2 * kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables, &unreachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - auto& unreachable = AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - unreachables[i] = unreachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - // Now start attaching new threads. - std::vector newMutators(kDefaultThreadCount); - std::vector> attachFutures(kDefaultThreadCount); - - for (int i = 0; i < kDefaultThreadCount; ++i) { - attachFutures[i] = newMutators[i].Execute([&gcDone, i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { - expandRootSet(threadData, mutator, i + kDefaultThreadCount); - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - }); - } - - // All the other threads are stopping at safe points. - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&gcDone](mm::ThreadData& threadData, Mutator& mutator) { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - // Wait for the GC to be done. - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - // GC will be completed first - for (auto& future : gcFutures) { - future.wait(); - } - - // Only then will the new threads be allowed to attach. - for (auto& future : attachFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (int i = 0; i < kDefaultThreadCount; ++i) { - expectedAlive.push_back(globals[i]); - expectedAlive.push_back(locals[i]); - expectedAlive.push_back(reachables[i]); - expectedAlive.push_back(globals[kDefaultThreadCount + i]); - expectedAlive.push_back(locals[kDefaultThreadCount + i]); - expectedAlive.push_back(reachables[kDefaultThreadCount + i]); - // Unreachables for new threads were not collected. - expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); - } - - // Force mutators to publish their internal heaps - // Really only needed for legacy allocators. - std::vector> publishFutures; - for (auto& mutator: mutators) { - publishFutures.emplace_back( - mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); - } - for (auto& mutator: newMutators) { - publishFutures.emplace_back( - mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); - } - for (auto& future : publishFutures) { - future.wait(); - } - - // All threads see the same alive objects, enough to check a single mutator. - EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); -} - -TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { - std::vector mutators(2); - std::atomic*> object1 = nullptr; - std::atomic weak = nullptr; - std::atomic done = false; - auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { - GlobalObjectHolder global1{threadData}; - auto& object1_local = AllocateObject(threadData); - object1 = &object1_local; - global1->field1 = object1_local.header(); - while (weak.load() == nullptr) - ; - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(object1_local.header()), false); - ASSERT_THAT(gc::isMarked(weak.load()->header()), false); - ASSERT_THAT(weak.load()->get(), object1_local.header()); - - global1->field1 = nullptr; - - EXPECT_CALL(finalizerHook(), Call(weak.load()->header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); - done = true; - }); - - auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { - while (object1.load() == nullptr) {} - ObjHolder holder; - auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot()); - weak = &weak_local; - *holder.slot() = nullptr; - while (!done) mm::safePoint(threadData); - }); - - f0.wait(); - f1.wait(); -} - -INSTANTIATE_TEST_SUITE_P(, - ConcurrentMarkAndSweepTest, - testing::Values( - ParallelismOptions{kDefaultThreadCount * 3, false, 0}, - ParallelismOptions{kDefaultThreadCount * 3, true, 0} -#if !__has_feature(thread_sanitizer) // TODO: Fix auxilary threads with tsan. - , ParallelismOptions{kDefaultThreadCount * 3, false, kDefaultThreadCount}, - ParallelismOptions{kDefaultThreadCount * 3, true, kDefaultThreadCount}, - - ParallelismOptions{kDefaultThreadCount / 2, true, kDefaultThreadCount}, - ParallelismOptions{kDefaultThreadCount / 2 * 3, true, kDefaultThreadCount} -#endif - ), - [] (const testing::TestParamInfo& paramInfo) { - using namespace std::string_literals; - auto base = "Mark"s; - auto parallelism = std::to_string(paramInfo.param.maxParallelism) + "Parallel"; - auto withMutators = paramInfo.param.cooperativeMutators ? "WithMutators" : ""; - auto withAux = paramInfo.param.auxGCThreads > 0 ? "WithGCThreads" : ""; - return base + parallelism + withMutators + withAux; - }); - +REGISTER_TYPED_TEST_SUITE_WITH_LISTS(TracingGCTest, TRACING_GC_TEST_LIST, CMSMultipleMutatorsWeak); +INSTANTIATE_TYPED_TEST_SUITE_P(CMS, TracingGCTest, ConcurrentMarkAndSweepTest); diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp similarity index 90% rename from kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweepTest.cpp rename to kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp index d17d1ddbd53..7c45030cd8e 100644 --- a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp @@ -1,9 +1,9 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ -#include "ParallelMarkConcurrentSweep.hpp" +#pragma once #include #include @@ -17,7 +17,6 @@ #include "AllocatorTestSupport.hpp" #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" -#include "GCImpl.hpp" #include "GlobalData.hpp" #include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" @@ -30,10 +29,6 @@ using namespace kotlin; -// These tests can only work if `GC` is `ParallelMarkConcurrentSweep`. - -namespace { - struct Payload { ObjHeader* field1; ObjHeader* field2; @@ -195,49 +190,23 @@ test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& thr return weakReference; } -struct ParallelismOptions { - std::size_t maxParallelism; - bool cooperativeMutators; - std::size_t auxGCThreads; -}; -class ParallelMarkConcurrentSweepTest : public testing::TestWithParam { +template +class TracingGCTest : public testing::Test { public: - - ParallelMarkConcurrentSweepTest() { - if (supportedConfiguration()) { - mm::GlobalData::Instance().gc().impl().gc().reconfigure(GetParam().maxParallelism, - GetParam().cooperativeMutators, - GetParam().auxGCThreads); - } - } - void SetUp() override { - if (!supportedConfiguration()) { - GTEST_SKIP() << "Unsupported parallelism configuration"; - } - } - - ~ParallelMarkConcurrentSweepTest() { - mm::GlobalsRegistry::Instance().ClearForTests(); - mm::SpecialRefRegistry::instance().clearForTests(); - mm::GlobalData::Instance().gc().ClearForTests(); - mm::GlobalData::Instance().allocator().clearForTests(); + impl_.SetUp(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } - private: - bool supportedConfiguration() const { - return !compiler::gcMarkSingleThreaded() || (!GetParam().cooperativeMutators && GetParam().auxGCThreads == 0); - } - + FixtureImpl impl_{}; FinalizerHooksTestSupport finalizerHooks_; }; -} // namespace +TYPED_TEST_SUITE_P(TracingGCTest); -TEST_P(ParallelMarkConcurrentSweepTest, RootSet) { +TYPED_TEST_P(TracingGCTest, RootSet) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global1{threadData}; GlobalObjectArrayHolder global2{threadData}; @@ -272,7 +241,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, RootSet) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, InterconnectedRootSet) { +TYPED_TEST_P(TracingGCTest, InterconnectedRootSet) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global1{threadData}; GlobalObjectArrayHolder global2{threadData}; @@ -318,7 +287,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, InterconnectedRootSet) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, FreeObjects) { +TYPED_TEST_P(TracingGCTest, FreeObjects) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); auto& object2 = AllocateObject(threadData); @@ -333,7 +302,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjects) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectsWithFinalizers) { +TYPED_TEST_P(TracingGCTest, FreeObjectsWithFinalizers) { RunInNewThread([this](mm::ThreadData& threadData) { auto& object1 = AllocateObjectWithFinalizer(threadData); auto& object2 = AllocateObjectWithFinalizer(threadData); @@ -342,15 +311,15 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectsWithFinalizers) { ASSERT_THAT(gc::isMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false); - EXPECT_CALL(finalizerHook(), Call(object1.header())); - EXPECT_CALL(finalizerHook(), Call(object2.header())); + EXPECT_CALL(this->finalizerHook(), Call(object1.header())); + EXPECT_CALL(this->finalizerHook(), Call(object2.header())); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); } -TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithFreeWeak) { +TYPED_TEST_P(TracingGCTest, FreeObjectWithFreeWeak) { RunInNewThread([this](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& { @@ -363,14 +332,14 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithFreeWeak) { ASSERT_THAT(gc::isMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); - EXPECT_CALL(finalizerHook(), Call(weak1.header())); + EXPECT_CALL(this->finalizerHook(), Call(weak1.header())); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); } -TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithHoldedWeak) { +TYPED_TEST_P(TracingGCTest, FreeObjectWithHoldedWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); StackObjectHolder stack{threadData}; @@ -389,7 +358,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithHoldedWeak) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, ObjectReferencedFromRootSet) { +TYPED_TEST_P(TracingGCTest, ObjectReferencedFromRootSet) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack{threadData}; @@ -429,7 +398,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, ObjectReferencedFromRootSet) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCycles) { +TYPED_TEST_P(TracingGCTest, ObjectsWithCycles) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack{threadData}; @@ -478,7 +447,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCycles) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCyclesAndFinalizers) { +TYPED_TEST_P(TracingGCTest, ObjectsWithCyclesAndFinalizers) { RunInNewThread([this](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack{threadData}; @@ -512,8 +481,8 @@ TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCyclesAndFinalizers) { ASSERT_THAT(gc::isMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false); - EXPECT_CALL(finalizerHook(), Call(object5.header())); - EXPECT_CALL(finalizerHook(), Call(object6.header())); + EXPECT_CALL(this->finalizerHook(), Call(object5.header())); + EXPECT_CALL(this->finalizerHook(), Call(object6.header())); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( @@ -529,7 +498,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCyclesAndFinalizers) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCyclesIntoRootSet) { +TYPED_TEST_P(TracingGCTest, ObjectsWithCyclesIntoRootSet) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack{threadData}; @@ -557,7 +526,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, ObjectsWithCyclesIntoRootSet) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, RunGCTwice) { +TYPED_TEST_P(TracingGCTest, RunGCTwice) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack{threadData}; @@ -607,7 +576,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, RunGCTwice) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, PermanentObjects) { +TYPED_TEST_P(TracingGCTest, PermanentObjects) { RunInNewThread([](mm::ThreadData& threadData) { GlobalPermanentObjectHolder global1{threadData}; GlobalObjectHolder global2{threadData}; @@ -629,7 +598,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, PermanentObjects) { }); } -TEST_P(ParallelMarkConcurrentSweepTest, SameObjectInRootSet) { +TYPED_TEST_P(TracingGCTest, SameObjectInRootSet) { RunInNewThread([](mm::ThreadData& threadData) { GlobalObjectHolder global{threadData}; StackObjectHolder stack(*global); @@ -715,7 +684,7 @@ private: } // namespace -TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsCollect) { +TYPED_TEST_P(TracingGCTest, MultipleMutatorsCollect) { std::vector mutators(kDefaultThreadCount); std::vector globals(kDefaultThreadCount); std::vector locals(kDefaultThreadCount); @@ -772,7 +741,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsCollect) { } } -TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAllCollect) { +TYPED_TEST_P(TracingGCTest, MultipleMutatorsAllCollect) { std::vector mutators(kDefaultThreadCount); std::vector globals(kDefaultThreadCount); std::vector locals(kDefaultThreadCount); @@ -801,7 +770,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAllCollect) { gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); // If GC starts before all thread executed line above, two gc will be run - // So we temporary switch threads to native state and then return them back after all GC runs are done + // So we temporarily switch threads to native state and then return them back after all GC runs are done SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative); })); } @@ -812,10 +781,10 @@ TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAllCollect) { for (int i = 0; i < kDefaultThreadCount; ++i) { mutators[i] - .Execute([](mm::ThreadData& threadData, Mutator& mutator) { + .Execute([](mm::ThreadData& threadData, Mutator& mutator) { SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kRunnable); }) - .wait(); + .wait(); } std::vector expectedAlive; @@ -834,7 +803,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAllCollect) { } } -TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { +TYPED_TEST_P(TracingGCTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { std::vector mutators(kDefaultThreadCount); std::vector globals(kDefaultThreadCount); std::vector locals(kDefaultThreadCount); @@ -905,7 +874,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsAddToRootSetAfterCollect } } -TEST_P(ParallelMarkConcurrentSweepTest, CrossThreadReference) { +TYPED_TEST_P(TracingGCTest, CrossThreadReference) { std::vector mutators(kDefaultThreadCount); std::vector globals(kDefaultThreadCount); std::vector locals(kDefaultThreadCount); @@ -972,107 +941,7 @@ TEST_P(ParallelMarkConcurrentSweepTest, CrossThreadReference) { } } -TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsWeaks) { - std::vector mutators(kDefaultThreadCount); - ObjHeader* globalRoot = nullptr; - test_support::RegularWeakReferenceImpl* weak = nullptr; - - mutators[0] - .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { - auto& global = mutator.AddGlobalRoot(); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - global->field1 = objectWeak.header(); - weak = &objectWeak; - globalRoot = global.header(); - }) - .wait(); - - // Make sure all mutators are initialized. - for (int i = 1; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - EXPECT_THAT(weak->get(), nullptr); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); - } -} - -TEST_P(ParallelMarkConcurrentSweepTest, MultipleMutatorsWeakNewObj) { - std::vector mutators(kDefaultThreadCount); - - // Make sure all mutators are initialized. - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - mm::safePoint(threadData); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - EXPECT_NE(objectWeak.get(), nullptr); - - auto& extraObj = *mm::ExtraObjectData::Get(object.header()); - extraObj.ClearRegularWeakReferenceImpl(); - extraObj.Uninstall(); - alloc::destroyExtraObjectData(extraObj); - - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } -} - -TEST_P(ParallelMarkConcurrentSweepTest, NewThreadsWhileRequestingCollection) { +TYPED_TEST_P(TracingGCTest, NewThreadsWhileRequestingCollection) { std::vector mutators(kDefaultThreadCount); std::vector globals(2 * kDefaultThreadCount); std::vector locals(2 * kDefaultThreadCount); @@ -1156,11 +1025,11 @@ TEST_P(ParallelMarkConcurrentSweepTest, NewThreadsWhileRequestingCollection) { // Force mutators to publish their internal heaps // Really only needed for legacy allocators. std::vector> publishFutures; - for (auto& mutator: mutators) { + for (auto& mutator : mutators) { publishFutures.emplace_back( mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } - for (auto& mutator: newMutators) { + for (auto& mutator : newMutators) { publishFutures.emplace_back( mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } @@ -1172,12 +1041,12 @@ TEST_P(ParallelMarkConcurrentSweepTest, NewThreadsWhileRequestingCollection) { EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); } -TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithFreeWeakReversedOrder) { +TYPED_TEST_P(TracingGCTest, FreeObjectWithFreeWeakReversedOrder) { std::vector mutators(2); std::atomic*> object1 = nullptr; std::atomic weak = nullptr; std::atomic done = false; - auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { + auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator&) { GlobalObjectHolder global1{threadData}; auto& object1_local = AllocateObject(threadData); object1 = &object1_local; @@ -1194,15 +1063,16 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithFreeWeakReversedOrder) { global1->field1 = nullptr; - EXPECT_CALL(finalizerHook(), Call(weak.load()->header())); + EXPECT_CALL(this->finalizerHook(), Call(weak.load()->header())); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); done = true; }); - auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { - while (object1.load() == nullptr) {} + auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator&) { + while (object1.load() == nullptr) { + } ObjHolder holder; auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot()); weak = &weak_local; @@ -1214,25 +1084,135 @@ TEST_P(ParallelMarkConcurrentSweepTest, FreeObjectWithFreeWeakReversedOrder) { f1.wait(); } -INSTANTIATE_TEST_SUITE_P(, - ParallelMarkConcurrentSweepTest, - testing::Values( - ParallelismOptions{kDefaultThreadCount * 3, false, 0}, - ParallelismOptions{kDefaultThreadCount * 3, true, 0} -#if !__has_feature(thread_sanitizer) // TODO: Fix auxilary threads with tsan. - , ParallelismOptions{kDefaultThreadCount * 3, false, kDefaultThreadCount}, - ParallelismOptions{kDefaultThreadCount * 3, true, kDefaultThreadCount}, +#define TRACING_GC_TEST_LIST \ + RootSet, \ + InterconnectedRootSet, \ + FreeObjects, \ + FreeObjectsWithFinalizers, \ + FreeObjectWithFreeWeak, \ + FreeObjectWithHoldedWeak, \ + ObjectReferencedFromRootSet, \ + ObjectsWithCycles, \ + ObjectsWithCyclesAndFinalizers, \ + ObjectsWithCyclesIntoRootSet, \ + RunGCTwice, \ + PermanentObjects, \ + SameObjectInRootSet, \ + MultipleMutatorsCollect, \ + MultipleMutatorsAllCollect, \ + MultipleMutatorsAddToRootSetAfterCollectionRequested, \ + CrossThreadReference, \ + NewThreadsWhileRequestingCollection, \ + FreeObjectWithFreeWeakReversedOrder - ParallelismOptions{kDefaultThreadCount / 2, true, kDefaultThreadCount}, - ParallelismOptions{kDefaultThreadCount / 2 * 3, true, kDefaultThreadCount} -#endif - ), - [] (const testing::TestParamInfo& paramInfo) { - using namespace std::string_literals; - auto base = "Mark"s; - auto parallelism = std::to_string(paramInfo.param.maxParallelism) + "Parallel"; - auto withMutators = paramInfo.param.cooperativeMutators ? "WithMutators" : ""; - auto withAux = paramInfo.param.auxGCThreads > 0 ? "WithGCThreads" : ""; - return base + parallelism + withMutators + withAux; - }); +template +class STWMarkGCTest : public TracingGCTest {}; +TYPED_TEST_SUITE_P(STWMarkGCTest); + +TYPED_TEST_P(STWMarkGCTest, MultipleMutatorsWeaks) { + std::vector mutators(kDefaultThreadCount); + ObjHeader* globalRoot = nullptr; + test_support::RegularWeakReferenceImpl* weak = nullptr; + + mutators[0] + .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { + auto& global = mutator.AddGlobalRoot(); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { + ObjHolder holder; + return InstallWeakReference(threadData, object.header(), holder.slot()); + })(); + global->field1 = objectWeak.header(); + weak = &objectWeak; + globalRoot = global.header(); + }) + .wait(); + + // Make sure all mutators are initialized. + for (int i = 1; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + std::vector> gcFutures; + auto epoch = mm::GlobalData::Instance().gc().Schedule(); + std::atomic gcDone = false; + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (auto& mutator : mutators) { + gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { + while (!gcDone.load(std::memory_order_relaxed)) { + mm::safePoint(threadData); + EXPECT_THAT(weak->get(), nullptr); + } + })); + } + + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); + gcDone.store(true, std::memory_order_relaxed); + + for (auto& future : gcFutures) { + future.wait(); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); + } +} + +TYPED_TEST_P(STWMarkGCTest, MultipleMutatorsWeakNewObj) { + std::vector mutators(kDefaultThreadCount); + + // Make sure all mutators are initialized. + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + std::vector> gcFutures; + auto epoch = mm::GlobalData::Instance().gc().Schedule(); + std::atomic gcDone = false; + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (auto& mutator : mutators) { + gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { + mm::safePoint(threadData); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { + ObjHolder holder; + return InstallWeakReference(threadData, object.header(), holder.slot()); + })(); + EXPECT_NE(objectWeak.get(), nullptr); + + auto& extraObj = *mm::ExtraObjectData::Get(object.header()); + extraObj.ClearRegularWeakReferenceImpl(); + extraObj.Uninstall(); + alloc::destroyExtraObjectData(extraObj); + + while (!gcDone.load(std::memory_order_relaxed)) { + mm::safePoint(threadData); + } + })); + } + + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); + gcDone.store(true, std::memory_order_relaxed); + + for (auto& future : gcFutures) { + future.wait(); + } +} + +#define STW_MARK_GC_TEST_LIST \ + MultipleMutatorsWeaks, \ + MultipleMutatorsWeakNewObj + +// expand lists before stringization in REGISTER_TYPED_TEST_SUITE_P +#define REGISTER_TYPED_TEST_SUITE_WITH_LISTS(SuiteName, ...) REGISTER_TYPED_TEST_SUITE_P(SuiteName, __VA_ARGS__) diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 71fb076fd21..609faa62473 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -1,201 +1,22 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ #include "SameThreadMarkAndSweep.hpp" -#include -#include -#include -#include -#include -#include - #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "AllocatorTestSupport.hpp" -#include "ExtraObjectData.hpp" -#include "FinalizerHooksTestSupport.hpp" #include "GCImpl.hpp" #include "GlobalData.hpp" -#include "ObjectOps.hpp" -#include "ObjectTestSupport.hpp" -#include "SafePoint.hpp" -#include "SingleThreadExecutor.hpp" -#include "TestSupport.hpp" -#include "ThreadData.hpp" -#include "WeakRef.hpp" +#include "TracingGCTest.hpp" using namespace kotlin; -// These tests can only work if `GC` is `SameThreadMarkAndSweep`. - namespace { -struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; - - static constexpr std::array kFields = { - &Payload::field1, - &Payload::field2, - &Payload::field3, - }; -}; - -test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; -test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; - -// TODO: Clean GlobalObjectHolder after it's gone. -class GlobalObjectHolder : private Pinned { -public: - explicit GlobalObjectHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateObject(&threadData, typeHolder.typeInfo(), &location_); - } - - GlobalObjectHolder(mm::ThreadData& threadData, ObjHeader* object) : location_(object) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::Object& operator*() { return test_support::Object::FromObjHeader(location_); } - test_support::Object& operator->() { return test_support::Object::FromObjHeader(location_); } - -private: - ObjHeader* location_ = nullptr; -}; - -// TODO: Clean GlobalPermanentObjectHolder after it's gone. -class GlobalPermanentObjectHolder : private Pinned { -public: - explicit GlobalPermanentObjectHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &global_); - global_->typeInfoOrMeta_ = setPointerBits(global_->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); - RuntimeAssert(global_->permanent(), "Must be permanent"); - } - - ObjHeader* header() { return global_; } - - test_support::Object& operator*() { return object_; } - test_support::Object& operator->() { return object_; } - -private: - test_support::Object object_{typeHolder.typeInfo()}; - ObjHeader* global_{object_.header()}; -}; - -// TODO: Clean GlobalObjectArrayHolder after it's gone. -class GlobalObjectArrayHolder : private Pinned { -public: - explicit GlobalObjectArrayHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateArray(&threadData, theArrayTypeInfo, 3, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } - test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } - - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } - -private: - ObjHeader* location_ = nullptr; -}; - -// TODO: Clean GlobalCharArrayHolder after it's gone. -class GlobalCharArrayHolder : private Pinned { -public: - explicit GlobalCharArrayHolder(mm::ThreadData& threadData) { - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); - mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, &location_); - } - - ObjHeader* header() { return location_; } - - test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } - test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(location_->array()); } - -private: - ObjHeader* location_ = nullptr; -}; - -class StackObjectHolder : private Pinned { -public: - explicit StackObjectHolder(mm::ThreadData& threadData) { mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder_.slot()); } - explicit StackObjectHolder(test_support::Object& object) : holder_(object.header()) {} - explicit StackObjectHolder(ObjHeader* object) : holder_(object) {} - - ObjHeader* header() { return holder_.obj(); } - - test_support::Object& operator*() { return test_support::Object::FromObjHeader(holder_.obj()); } - test_support::Object& operator->() { return test_support::Object::FromObjHeader(holder_.obj()); } - -private: - ObjHolder holder_; -}; - -class StackObjectArrayHolder : private Pinned { -public: - explicit StackObjectArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theArrayTypeInfo, 3, holder_.slot()); } - - ObjHeader* header() { return holder_.obj(); } - - test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } - test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } - - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } - -private: - ObjHolder holder_; -}; - -class StackCharArrayHolder : private Pinned { -public: - explicit StackCharArrayHolder(mm::ThreadData& threadData) { mm::AllocateArray(&threadData, theCharArrayTypeInfo, 3, holder_.slot()); } - - ObjHeader* header() { return holder_.obj(); } - - test_support::CharArray<3>& operator*() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } - test_support::CharArray<3>& operator->() { return test_support::CharArray<3>::FromArrayHeader(holder_.obj()->array()); } - -private: - ObjHolder holder_; -}; - -test_support::Object& AllocateObject(mm::ThreadData& threadData) { - ObjHolder holder; - mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder.slot()); - return test_support::Object::FromObjHeader(holder.obj()); -} - -test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threadData) { - ObjHolder holder; - mm::AllocateObject(&threadData, typeHolderWithFinalizer.typeInfo(), holder.slot()); - return test_support::Object::FromObjHeader(holder.obj()); -} - -std::vector Alive(mm::ThreadData& threadData) { - return alloc::test_support::allocatedObjects(threadData); -} - -test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { - mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); - auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); - auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); - weakReference->weakRef = static_cast(mm::WeakRef::create(objHeader)); - weakReference->referred = objHeader; - auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header()); - EXPECT_EQ(setWeakRef, weakReference.header()); - return weakReference; -} - -class SameThreadMarkAndSweepTest : public testing::Test { +class SameThreadMarkAndSweepTest { public: ~SameThreadMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); @@ -204,987 +25,13 @@ public: mm::GlobalData::Instance().allocator().clearForTests(); } - testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } - -private: - FinalizerHooksTestSupport finalizerHooks_; + void SetUp() {} }; } // namespace -TEST_F(SameThreadMarkAndSweepTest, RootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global1{threadData}; - GlobalObjectArrayHolder global2{threadData}; - GlobalCharArrayHolder global3{threadData}; - StackObjectHolder stack1{threadData}; - StackObjectArrayHolder stack2{threadData}; - StackCharArrayHolder stack3{threadData}; +REGISTER_TYPED_TEST_SUITE_WITH_LISTS(TracingGCTest, TRACING_GC_TEST_LIST); +INSTANTIATE_TYPED_TEST_SUITE_P(STMS, TracingGCTest, SameThreadMarkAndSweepTest); - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(global2.header()), false); - ASSERT_THAT(gc::isMarked(global3.header()), false); - ASSERT_THAT(gc::isMarked(stack1.header()), false); - ASSERT_THAT(gc::isMarked(stack2.header()), false); - ASSERT_THAT(gc::isMarked(stack3.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(gc::isMarked(global1.header()), false); - EXPECT_THAT(gc::isMarked(global2.header()), false); - EXPECT_THAT(gc::isMarked(global3.header()), false); - EXPECT_THAT(gc::isMarked(stack1.header()), false); - EXPECT_THAT(gc::isMarked(stack2.header()), false); - EXPECT_THAT(gc::isMarked(stack3.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global1{threadData}; - GlobalObjectArrayHolder global2{threadData}; - GlobalCharArrayHolder global3{threadData}; - StackObjectHolder stack1{threadData}; - StackObjectArrayHolder stack2{threadData}; - StackCharArrayHolder stack3{threadData}; - - global1->field1 = stack1.header(); - global1->field2 = global1.header(); - global1->field3 = global2.header(); - global2[0] = global1.header(); - global2[1] = global3.header(); - stack1->field1 = global1.header(); - stack1->field2 = stack1.header(); - stack1->field3 = stack2.header(); - stack2[0] = stack1.header(); - stack2[1] = stack3.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(global2.header()), false); - ASSERT_THAT(gc::isMarked(global3.header()), false); - ASSERT_THAT(gc::isMarked(stack1.header()), false); - ASSERT_THAT(gc::isMarked(stack2.header()), false); - ASSERT_THAT(gc::isMarked(stack3.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(gc::isMarked(global1.header()), false); - EXPECT_THAT(gc::isMarked(global2.header()), false); - EXPECT_THAT(gc::isMarked(global3.header()), false); - EXPECT_THAT(gc::isMarked(stack1.header()), false); - EXPECT_THAT(gc::isMarked(stack2.header()), false); - EXPECT_THAT(gc::isMarked(stack3.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, FreeObjects) { - RunInNewThread([](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { - RunInNewThread([this](mm::ThreadData& threadData) { - auto& object1 = AllocateObjectWithFinalizer(threadData); - auto& object2 = AllocateObjectWithFinalizer(threadData); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - EXPECT_CALL(finalizerHook(), Call(object1.header())); - EXPECT_CALL(finalizerHook(), Call(object2.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { - RunInNewThread([this](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object1.header(), holder.slot()); - })(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(weak1.header()), false); - ASSERT_THAT(weak1.get(), object1.header()); - - EXPECT_CALL(finalizerHook(), Call(weak1.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { - RunInNewThread([](mm::ThreadData& threadData) { - auto& object1 = AllocateObject(threadData); - StackObjectHolder stack{threadData}; - auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(weak1.header()), false); - ASSERT_THAT(weak1.get(), object1.header()); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(gc::isMarked(weak1.header()), false); - EXPECT_THAT(weak1.get(), nullptr); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - auto& object5 = AllocateObject(threadData); - auto& object6 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { - RunInNewThread([this](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObjectWithFinalizer(threadData); - auto& object2 = AllocateObjectWithFinalizer(threadData); - auto& object3 = AllocateObjectWithFinalizer(threadData); - auto& object4 = AllocateObjectWithFinalizer(threadData); - auto& object5 = AllocateObjectWithFinalizer(threadData); - auto& object6 = AllocateObjectWithFinalizer(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - EXPECT_CALL(finalizerHook(), Call(object5.header())); - EXPECT_CALL(finalizerHook(), Call(object6.header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = global.header(); - stack->field1 = object2.header(); - object2->field1 = stack.header(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack{threadData}; - auto& object1 = AllocateObject(threadData); - auto& object2 = AllocateObject(threadData); - auto& object3 = AllocateObject(threadData); - auto& object4 = AllocateObject(threadData); - auto& object5 = AllocateObject(threadData); - auto& object6 = AllocateObject(threadData); - - global->field1 = object1.header(); - object1->field1 = object2.header(); - object2->field1 = object1.header(); - stack->field1 = object3.header(); - object3->field1 = object4.header(); - object4->field1 = object3.header(); - object5->field1 = object6.header(); - object6->field1 = object5.header(); - - ASSERT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), - object5.header(), object6.header())); - ASSERT_THAT(gc::isMarked(global.header()), false); - ASSERT_THAT(gc::isMarked(stack.header()), false); - ASSERT_THAT(gc::isMarked(object1.header()), false); - ASSERT_THAT(gc::isMarked(object2.header()), false); - ASSERT_THAT(gc::isMarked(object3.header()), false); - ASSERT_THAT(gc::isMarked(object4.header()), false); - ASSERT_THAT(gc::isMarked(object5.header()), false); - ASSERT_THAT(gc::isMarked(object6.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT( - Alive(threadData), - testing::UnorderedElementsAre( - global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(stack.header()), false); - EXPECT_THAT(gc::isMarked(object1.header()), false); - EXPECT_THAT(gc::isMarked(object2.header()), false); - EXPECT_THAT(gc::isMarked(object3.header()), false); - EXPECT_THAT(gc::isMarked(object4.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalPermanentObjectHolder global1{threadData}; - GlobalObjectHolder global2{threadData}; - test_support::Object permanentObject{typeHolder.typeInfo()}; - permanentObject.header()->typeInfoOrMeta_ = - setPointerBits(permanentObject.header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); - RuntimeAssert(permanentObject.header()->permanent(), "Must be permanent"); - - global1->field1 = permanentObject.header(); - global2->field1 = global1.header(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(gc::isMarked(global2.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(gc::isMarked(global2.header()), false); - }); -} - -TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) { - RunInNewThread([](mm::ThreadData& threadData) { - GlobalObjectHolder global{threadData}; - StackObjectHolder stack(*global); - auto& object = AllocateObject(threadData); - - global->field1 = object.header(); - - ASSERT_THAT(global.header(), stack.header()); - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(object.header()), false); - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(gc::isMarked(global.header()), false); - EXPECT_THAT(gc::isMarked(object.header()), false); - }); -} - -namespace { - -class Mutator : private Pinned { -public: - template - [[nodiscard]] std::future Execute(F&& f) { - return executor_.execute( - [this, f = std::forward(f)] { f(*executor_.context().memory_->memoryState()->GetThreadData(), *this); }); - } - - StackObjectHolder& AddStackRoot() { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddStackRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData()); - auto& holderRef = *holder; - context.stackRoots_.push_back(std::move(holder)); - return holderRef; - } - - StackObjectHolder& AddStackRoot(ObjHeader* object) { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddStackRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(object); - auto& holderRef = *holder; - context.stackRoots_.push_back(std::move(holder)); - return holderRef; - } - - GlobalObjectHolder& AddGlobalRoot() { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddGlobalRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData()); - auto& holderRef = *holder; - context.globalRoots_.push_back(std::move(holder)); - return holderRef; - } - - GlobalObjectHolder& AddGlobalRoot(ObjHeader* object) { - RuntimeAssert(std::this_thread::get_id() == executor_.threadId(), "AddGlobalRoot can only be called in the mutator thread"); - auto& context = executor_.context(); - auto holder = std::make_unique(*context.memory_->memoryState()->GetThreadData(), object); - auto& holderRef = *holder; - context.globalRoots_.push_back(std::move(holder)); - return holderRef; - } - - std::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } - -private: - struct Context { - std::unique_ptr memory_; - std::vector> stackRoots_; - std::vector> globalRoots_; - - Context() : memory_(std::make_unique()) { - // SingleThreadExecutor must work in the runnable state, so that GC does not collect between tasks. - AssertThreadState(memory_->memoryState(), ThreadState::kRunnable); - } - }; - - SingleThreadExecutor executor_; -}; - -} // namespace - -TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - std::atomic gcDone = false; - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAllCollect) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - // If GC starts before all thread executed line above, two gc will be run - // So we temporary switch threads to native state and then return them back after all GC runs are done - SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative); - })); - } - - for (auto& future : gcFutures) { - future.wait(); - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([](mm::ThreadData& threadData, Mutator& mutator) { - SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kRunnable); - }) - .wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(kDefaultThreadCount); - - auto allocateInHeap = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = AllocateObject(threadData); - auto& local = AllocateObject(threadData); - auto& reachable = AllocateObject(threadData); - AllocateObject(threadData); - - local->field1 = reachable.header(); - - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - }; - - auto expandRootSet = [&globals, &locals](mm::ThreadData& threadData, Mutator& mutator, int i) { - mutator.AddGlobalRoot(globals[i]); - mutator.AddStackRoot(locals[i]); - }; - - // Allocate everything in heap before scheduling the GC. - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([allocateInHeap, i](mm::ThreadData& threadData, Mutator& mutator) { allocateInHeap(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - gcFutures.emplace_back(mutators[i].Execute([&gcDone, i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { - expandRootSet(threadData, mutator, i); - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(kDefaultThreadCount); - std::vector locals(kDefaultThreadCount); - std::vector reachables(2 * kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable1 = AllocateObject(threadData); - auto& reachable2 = AllocateObject(threadData); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[2 * i] = reachable1.header(); - reachables[2 * i + 1] = reachable2.header(); - - // Expected to be run consequtively, so `reachables` for `j < i` are set. - if (i != 0) { - global->field1 = reachables[2 * (i - 1)]; - local->field1 = reachables[2 * (i - 1) + 1]; - } - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - // `expandRootSet` is expected to be run consequtively for each thread, so `.wait()` is required below. - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - - std::atomic gcDone = false; - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (auto& global : globals) { - expectedAlive.push_back(global); - } - for (auto& local : locals) { - expectedAlive.push_back(local); - } - // The last two are in fact unreachable. Their absence allows us to check that GC was in fact performed. - reachables.pop_back(); - reachables.pop_back(); - for (auto& reachable : reachables) { - expectedAlive.push_back(reachable); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } -} - -TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { - std::vector mutators(kDefaultThreadCount); - ObjHeader* globalRoot = nullptr; - test_support::RegularWeakReferenceImpl* weak = nullptr; - - mutators[0] - .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { - auto& global = mutator.AddGlobalRoot(); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - global->field1 = objectWeak.header(); - weak = &objectWeak; - globalRoot = global.header(); - }) - .wait(); - - // Make sure all mutators are initialized. - for (int i = 1; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - EXPECT_THAT(weak->get(), nullptr); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); - } -} - -TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeakNewObj) { - std::vector mutators(kDefaultThreadCount); - - // Make sure all mutators are initialized. - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - mm::safePoint(threadData); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - EXPECT_NE(objectWeak.get(), nullptr); - - auto& extraObj = *mm::ExtraObjectData::Get(object.header()); - extraObj.ClearRegularWeakReferenceImpl(); - extraObj.Uninstall(); - alloc::destroyExtraObjectData(extraObj); - - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } -} - -TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { - std::vector mutators(kDefaultThreadCount); - std::vector globals(2 * kDefaultThreadCount); - std::vector locals(2 * kDefaultThreadCount); - std::vector reachables(2 * kDefaultThreadCount); - std::vector unreachables(2 * kDefaultThreadCount); - - auto expandRootSet = [&globals, &locals, &reachables, &unreachables](mm::ThreadData& threadData, Mutator& mutator, int i) { - auto& global = mutator.AddGlobalRoot(); - auto& local = mutator.AddStackRoot(); - auto& reachable = AllocateObject(threadData); - auto& unreachable = AllocateObject(threadData); - local->field1 = reachable.header(); - globals[i] = global.header(); - locals[i] = local.header(); - reachables[i] = reachable.header(); - unreachables[i] = unreachable.header(); - }; - - for (int i = 0; i < kDefaultThreadCount; ++i) { - mutators[i] - .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) - .wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) { - } - - // Now start attaching new threads. - std::vector newMutators(kDefaultThreadCount); - std::vector> attachFutures(kDefaultThreadCount); - - for (int i = 0; i < kDefaultThreadCount; ++i) { - attachFutures[i] = newMutators[i].Execute([&gcDone, i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { - expandRootSet(threadData, mutator, i + kDefaultThreadCount); - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - }); - } - - // All the other threads are stopping at safe points. - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&gcDone](mm::ThreadData& threadData, Mutator& mutator) { - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - } - })); - } - - // Wait for the GC to be done. - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - // GC will be completed first - for (auto& future : gcFutures) { - future.wait(); - } - - // Only then will the new threads be allowed to attach. - for (auto& future : attachFutures) { - future.wait(); - } - - std::vector expectedAlive; - for (int i = 0; i < kDefaultThreadCount; ++i) { - expectedAlive.push_back(globals[i]); - expectedAlive.push_back(locals[i]); - expectedAlive.push_back(reachables[i]); - expectedAlive.push_back(globals[kDefaultThreadCount + i]); - expectedAlive.push_back(locals[kDefaultThreadCount + i]); - expectedAlive.push_back(reachables[kDefaultThreadCount + i]); - // Unreachables for new threads were not collected. - expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); - } - - // Force mutators to publish their internal heaps - std::vector> publishFutures; - for (auto& mutator: mutators) { - publishFutures.emplace_back( - mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); - } - for (auto& mutator: newMutators) { - publishFutures.emplace_back( - mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); - } - for (auto& future : publishFutures) { - future.wait(); - } - - // All threads see the same alive objects, enough to check a single mutator. - EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); -} - - -TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { - std::vector mutators(2); - std::atomic*> object1 = nullptr; - std::atomic weak = nullptr; - std::atomic done = false; - auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { - GlobalObjectHolder global1{threadData}; - auto& object1_local = AllocateObject(threadData); - object1 = &object1_local; - global1->field1 = object1_local.header(); - while (weak.load() == nullptr) - ; - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(gc::isMarked(global1.header()), false); - ASSERT_THAT(gc::isMarked(object1_local.header()), false); - ASSERT_THAT(gc::isMarked(weak.load()->header()), false); - ASSERT_THAT(weak.load()->get(), object1_local.header()); - - global1->field1 = nullptr; - - EXPECT_CALL(finalizerHook(), Call(weak.load()->header())); - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); - - EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); - done = true; - }); - - auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { - while (object1.load() == nullptr) {} - ObjHolder holder; - auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot()); - weak = &weak_local; - *holder.slot() = nullptr; - while (!done) mm::safePoint(threadData); - }); - - f0.wait(); - f1.wait(); -} +REGISTER_TYPED_TEST_SUITE_WITH_LISTS(STWMarkGCTest, STW_MARK_GC_TEST_LIST); +INSTANTIATE_TYPED_TEST_SUITE_P(STMS, STWMarkGCTest, SameThreadMarkAndSweepTest); diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp index 4460b398bea..4230aedc019 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp @@ -3,6 +3,8 @@ * that can be found in the LICENSE file. */ +#pragma once + #include #include #include diff --git a/kotlin-native/runtime/src/mm/cpp/TestSupport.hpp b/kotlin-native/runtime/src/mm/cpp/TestSupport.hpp index 4a6c06456d1..aea023d749b 100644 --- a/kotlin-native/runtime/src/mm/cpp/TestSupport.hpp +++ b/kotlin-native/runtime/src/mm/cpp/TestSupport.hpp @@ -3,6 +3,8 @@ * that can be found in the LICENSE file. */ +#pragma once + #include "../../main/cpp/TestSupport.hpp" #include