From 63e65a482c9a0b67b73c26e9a461dd0fad058c6a Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Wed, 1 Mar 2023 10:19:40 +0100 Subject: [PATCH] [K/N] Move GC schedulers into modules Additionally, - rename disabled into manual - rename with_timer into adaptive - remove on_safepoints - store schedulers in GlobalData, ThreadData directly --- .../kotlin/backend/konan/GCSchedulerType.kt | 14 +- .../kotlin/backend/konan/KonanConfig.kt | 15 + .../kotlin/backend/konan/llvm/IrToBitcode.kt | 1 - kotlin-native/runtime/build.gradle.kts | 84 +- .../src/custom_alloc/cpp/CustomAllocator.cpp | 2 +- .../src/custom_alloc/cpp/CustomAllocator.hpp | 4 +- .../custom_alloc/cpp/CustomAllocatorTest.cpp | 26 +- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 9 +- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 13 +- .../runtime/src/gc/cms/cpp/GCImpl.cpp | 30 +- .../runtime/src/gc/cms/cpp/GCImpl.hpp | 16 +- .../runtime/src/gc/common/cpp/GC.hpp | 10 +- .../runtime/src/gc/common/cpp/GCScheduler.cpp | 52 -- .../src/gc/common/cpp/GCSchedulerImpl.hpp | 231 ----- .../src/gc/common/cpp/GCSchedulerTest.cpp | 877 ------------------ .../src/gc/common/cpp/MarkAndSweepUtils.hpp | 2 +- .../runtime/src/gc/noop/cpp/GCImpl.cpp | 15 +- .../runtime/src/gc/noop/cpp/GCImpl.hpp | 9 +- .../runtime/src/gc/noop/cpp/NoOpGC.hpp | 13 +- .../runtime/src/gc/stms/cpp/GCImpl.cpp | 30 +- .../runtime/src/gc/stms/cpp/GCImpl.hpp | 12 +- .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 8 +- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 10 +- .../stms/cpp/SameThreadMarkAndSweepTest.cpp | 12 +- .../adaptive/cpp/GCSchedulerImpl.cpp | 22 + .../adaptive/cpp/GCSchedulerImpl.hpp | 66 ++ .../adaptive/cpp/GCSchedulerImplTest.cpp | 288 ++++++ .../aggressive/cpp/GCSchedulerImpl.cpp | 24 + .../aggressive/cpp/GCSchedulerImpl.hpp | 52 ++ .../aggressive/cpp/GCSchedulerImplTest.cpp | 75 ++ .../aggressive/cpp/SafePointTracker.hpp | 53 ++ .../aggressive/cpp/SafePointTrackerTest.cpp | 89 ++ .../gcScheduler/common/cpp/GCScheduler.cpp | 6 + .../common/cpp/GCScheduler.hpp | 73 +- .../common/cpp/GCSchedulerConfig.hpp | 38 + .../common/cpp/GCSchedulerTest.cpp | 263 ++++++ .../common/cpp/GCSchedulerTestSupport.cpp | 6 + .../common/cpp/GCSchedulerTestSupport.hpp | 24 + .../common/cpp/HeapGrowthController.hpp | 60 ++ .../common/cpp/RegularIntervalPacer.hpp | 36 + .../manual/cpp/GCSchedulerImpl.cpp | 12 + .../manual/cpp/GCSchedulerImpl.hpp | 23 + .../src/main/cpp/CompilerConstants.hpp | 14 - .../runtime/src/mm/cpp/GlobalData.hpp | 4 +- kotlin-native/runtime/src/mm/cpp/Memory.cpp | 34 +- .../runtime/src/mm/cpp/ThreadData.hpp | 6 +- .../test_support/cpp/CompilerGenerated.cpp | 1 - 47 files changed, 1350 insertions(+), 1414 deletions(-) delete mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp delete mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCSchedulerImpl.hpp delete mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTracker.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTrackerTest.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.cpp rename kotlin-native/runtime/src/{gc => gcScheduler}/common/cpp/GCScheduler.hpp (53%) create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/common/cpp/RegularIntervalPacer.hpp create mode 100644 kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp create mode 100644 kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt index 658fb57945e..bdf4f03c9d1 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt @@ -6,12 +6,12 @@ package org.jetbrains.kotlin.backend.konan // Must match `GCSchedulerType` in CompilerConstants.hpp -enum class GCSchedulerType(val value: Int, val deprecatedWithReplacement: GCSchedulerType? = null) { - MANUAL(0), - ADAPTIVE(1), - AGGRESSIVE(3), +enum class GCSchedulerType(val deprecatedWithReplacement: GCSchedulerType? = null) { + MANUAL, + ADAPTIVE, + AGGRESSIVE, // Deprecated: - DISABLED(0, deprecatedWithReplacement = MANUAL), - WITH_TIMER(1, deprecatedWithReplacement = ADAPTIVE), - ON_SAFE_POINTS(2, deprecatedWithReplacement = ADAPTIVE), + DISABLED(deprecatedWithReplacement = MANUAL), + WITH_TIMER(deprecatedWithReplacement = ADAPTIVE), + ON_SAFE_POINTS(deprecatedWithReplacement = ADAPTIVE), } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 4c77daa4c65..eecc9489a31 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -272,6 +272,21 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val runtimeNativeLibraries: List = mutableListOf().apply { if (debug) add("debug.bc") add("common_gc.bc") + add("common_gcScheduler.bc") + when (gcSchedulerType) { + GCSchedulerType.MANUAL -> { + add("manual_gcScheduler.bc") + } + GCSchedulerType.ADAPTIVE -> { + add("adaptive_gcScheduler.bc") + } + GCSchedulerType.AGGRESSIVE -> { + add("aggressive_gcScheduler.bc") + } + GCSchedulerType.DISABLED, GCSchedulerType.WITH_TIMER, GCSchedulerType.ON_SAFE_POINTS -> { + throw IllegalStateException("Deprecated options must have already been handled") + } + } if (allocationMode == AllocationMode.CUSTOM) { add("experimental_memory_manager_custom.bc") add("concurrent_ms_gc_custom.bc") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index a8bee200702..551c33c673e 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -3000,7 +3000,6 @@ internal fun NativeGenerationState.generateRuntimeConstantsModule() : LLVMModule setRuntimeConstGlobal("Kotlin_runtimeLogs", runtimeLogs) setRuntimeConstGlobal("Kotlin_freezingEnabled", llvm.constInt32(if (config.freezing.enableFreezeAtRuntime) 1 else 0)) setRuntimeConstGlobal("Kotlin_freezingChecksEnabled", llvm.constInt32(if (config.freezing.enableFreezeChecks) 1 else 0)) - setRuntimeConstGlobal("Kotlin_gcSchedulerType", llvm.constInt32(config.gcSchedulerType.value)) return llvmModule } diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 813a275b9a1..b41a859143b 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -158,7 +158,7 @@ bitcode { } module("custom_alloc") { - headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gc/cms/cpp")) + headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/cms/cpp")) sourceSets { main {} test {} @@ -253,7 +253,7 @@ bitcode { module("experimental_memory_manager") { srcRoot.set(layout.projectDirectory.dir("src/mm")) - headersDirs.from(files("src/gc/common/cpp", "src/main/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp")) sourceSets { main {} testFixtures {} @@ -265,7 +265,7 @@ bitcode { module("experimental_memory_manager_custom") { srcRoot.set(layout.projectDirectory.dir("src/mm")) - headersDirs.from(files("src/gc/common/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp", "src/custom_alloc/cpp")) sourceSets { main {} testFixtures {} @@ -279,7 +279,7 @@ bitcode { module("common_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/common")) - headersDirs.from(files("src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -290,7 +290,7 @@ bitcode { module("noop_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/noop")) - headersDirs.from(files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} testFixtures {} @@ -301,7 +301,7 @@ bitcode { module("same_thread_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/stms")) - headersDirs.from(files("src/gc/stms/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} testFixtures {} @@ -313,7 +313,7 @@ bitcode { module("concurrent_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/cms")) - headersDirs.from(files("src/gc/cms/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} testFixtures {} @@ -325,7 +325,7 @@ bitcode { module("concurrent_ms_gc_custom") { srcRoot.set(layout.projectDirectory.dir("src/gc/cms")) - headersDirs.from(files("src/gc/cms/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) sourceSets { main {} testFixtures {} @@ -337,33 +337,87 @@ bitcode { onlyIf { target.supportsThreads() } } + module("common_gcScheduler") { + srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/common")) + headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + testFixtures {} + test {} + } + + onlyIf { target.supportsThreads() } + } + + module("manual_gcScheduler") { + srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/manual")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + } + + onlyIf { target.supportsThreads() } + } + + module("adaptive_gcScheduler") { + srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/adaptive")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + test {} + } + + onlyIf { target.supportsThreads() } + } + + module("aggressive_gcScheduler") { + srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/aggressive")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + test {} + } + + onlyIf { target.supportsThreads() } + } + testsGroup("custom_alloc_runtime_tests") { testedModules.addAll("custom_alloc") - testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "concurrent_ms_gc", "objc") + testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "objc") } testsGroup("experimentalMM_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "same_thread_ms_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "opt_alloc", "objc") } testsGroup("experimentalMM_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "same_thread_ms_gc", "std_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc") } testsGroup("experimentalMM_cms_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc") } testsGroup("experimentalMM_cms_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "concurrent_ms_gc", "std_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc") } testsGroup("experimentalMM_noop_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "noop_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc") } testsGroup("experimentalMM_noop_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "noop_gc", "std_alloc", "objc") + testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc") + } + + testsGroup("aggressive_gcScheduler_runtime_tests") { + testedModules.addAll("aggressive_gcScheduler") + testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc") + } + + testsGroup("adaptive_gcScheduler_runtime_tests") { + testedModules.addAll("adaptive_gcScheduler") + testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc") } } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index dff6abbf00d..91d31a270d4 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -42,7 +42,7 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); } -CustomAllocator::CustomAllocator(Heap& heap, gc::GCSchedulerThreadData& gcScheduler) noexcept : +CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : heap_(heap), gcScheduler_(gcScheduler), nextFitPage_(nullptr), extraObjectPage_(nullptr) { CustomAllocInfo("CustomAllocator::CustomAllocator(heap)"); memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index 807ab7b24f8..a6cbcb33832 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -21,7 +21,7 @@ namespace kotlin::alloc { class CustomAllocator { public: - explicit CustomAllocator(Heap& heap, gc::GCSchedulerThreadData& gcScheduler) noexcept; + explicit CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept; ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; @@ -45,7 +45,7 @@ private: uint8_t* AllocateInFixedBlockPage(uint32_t cellCount) noexcept; Heap& heap_; - gc::GCSchedulerThreadData& gcScheduler_; + gcScheduler::GCSchedulerThreadData& gcScheduler_; NextFitPage* nextFitPage_; FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; ExtraObjectPage* extraObjectPage_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp index aa386fba3e9..e30008ed63a 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp @@ -28,8 +28,8 @@ TEST(CustomAllocTest, SmallAllocNonNull) { fakeTypes[i] = {.typeInfo_ = &fakeTypes[i], .instanceSize_ = 8 * i, .flags_ = 0}; } Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); ObjHeader* obj[N]; for (int i = 1; i < N; ++i) { @@ -43,8 +43,8 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { const int N = FIXED_BLOCK_PAGE_CELL_COUNT / FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; for (int blocks = MIN_BLOCK_SIZE; blocks < FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blocks) { Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; uint8_t* first = reinterpret_cast(ca.CreateObject(&fakeType)); @@ -58,8 +58,8 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { TEST(CustomAllocTest, FixedBlockPageThreshold) { Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10; const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10; @@ -71,8 +71,8 @@ TEST(CustomAllocTest, FixedBlockPageThreshold) { TEST(CustomAllocTest, NextFitPageThreshold) { Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10; const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10; @@ -85,9 +85,9 @@ TEST(CustomAllocTest, NextFitPageThreshold) { TEST(CustomAllocTest, TwoAllocatorsDifferentPages) { for (int blocks = MIN_BLOCK_SIZE; blocks < 2000; ++blocks) { Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData1(config, [](auto&) {}); - kotlin::gc::GCSchedulerThreadData schedulerData2(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData1(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerThreadData schedulerData2(config, [](auto&) {}); CustomAllocator ca1(heap, schedulerData1); CustomAllocator ca2(heap, schedulerData2); TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; @@ -102,8 +102,8 @@ using Data = typename kotlin::mm::ExtraObjectData; TEST(CustomAllocTest, AllocExtraObjectNonNullZeroed) { Heap heap; - kotlin::gc::GCSchedulerConfig config; - kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); + kotlin::gcScheduler::GCSchedulerConfig config; + kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); for (int i = 1; i < 10; ++i) { uint8_t* obj = reinterpret_cast(ca.CreateExtraObject()); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 6ddaf931974..ac3c6d81630 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -109,21 +109,16 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { #ifndef CUSTOM_ALLOCATOR gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : + mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), #else -gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(GCScheduler& gcScheduler) noexcept : +gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept : #endif gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) { GCHandle::getByEpoch(epoch).finalizersDone(); state_.finalized(epoch); }) { - gcScheduler_.SetScheduleGC([this]() NO_INLINE { - // This call acquires a lock, so we need to ensure that we're in the safe state. - NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); - state_.schedule(); - }); gcThread_ = ScopedThread(ScopedThread::attributes().name("GC thread"), [this] { while (true) { auto epoch = state_.waitScheduled(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 5f0ec698697..c818b262012 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -76,7 +76,8 @@ public: using Allocator = AllocatorWithGC; - explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : + explicit ThreadData( + ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : gc_(gc), threadData_(threadData), gcScheduler_(gcScheduler) {} ~ThreadData() = default; @@ -96,7 +97,7 @@ public: friend ConcurrentMarkAndSweep; ConcurrentMarkAndSweep& gc_; mm::ThreadData& threadData_; - GCSchedulerThreadData& gcScheduler_; + gcScheduler::GCSchedulerThreadData& gcScheduler_; std::atomic marking_; }; @@ -111,9 +112,9 @@ public: #endif #ifdef CUSTOM_ALLOCATOR - explicit ConcurrentMarkAndSweep(GCScheduler& scheduler) noexcept; + explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler) noexcept; #else - ConcurrentMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& scheduler) noexcept; + ConcurrentMarkAndSweep(mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& scheduler) noexcept; #endif ~ConcurrentMarkAndSweep(); @@ -129,6 +130,8 @@ public: alloc::Heap& heap() noexcept { return heap_; } #endif + void Schedule() noexcept { state_.schedule(); } + private: void PerformFullGC(int64_t epoch) noexcept; @@ -137,7 +140,7 @@ private: #else alloc::Heap heap_; #endif - GCScheduler& gcScheduler_; + gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; ScopedThread gcThread_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 194cf29d00b..c3c1c62a69b 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -13,25 +13,17 @@ using namespace kotlin; -namespace { - -ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept { - threadData.impl().gcScheduler().OnSafePointRegular(weight); - mm::SuspendIfRequested(); -} - -} // namespace - -gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : + impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept { - SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight); + mm::SuspendIfRequested(); } ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight); + mm::SuspendIfRequested(); } void gc::GC::ThreadData::Schedule() noexcept { @@ -74,15 +66,11 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI #endif } -void gc::GC::ThreadData::OnStoppedForGC() noexcept { - impl_->gcScheduler().OnStoppedForGC(); -} - void gc::GC::ThreadData::OnSuspendForGC() noexcept { impl_->gc().OnSuspendForGC(); } -gc::GC::GC() noexcept : impl_(std_support::make_unique()) {} +gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} gc::GC::~GC() = default; @@ -99,10 +87,6 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } -gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { - return impl_->gcScheduler().config(); -} - void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR @@ -137,3 +121,7 @@ ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) n ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { gc::internal::processFieldInMark(state, field); } + +void gc::GC::Schedule() noexcept { + impl_->gc().Schedule(); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 86499eaa0cc..68c94572caa 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -21,37 +21,34 @@ using GCImpl = ConcurrentMarkAndSweep; class GC::Impl : private Pinned { public: #ifdef CUSTOM_ALLOCATOR - Impl() noexcept : gc_(gcScheduler_) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {} #else - Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {} #endif #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } #endif - GCScheduler& gcScheduler() noexcept { return gcScheduler_; } GCImpl& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; #endif - GCScheduler gcScheduler_; GCImpl gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : - gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), - gc_(gc.impl_->gc(), threadData, gcScheduler_), + Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : + gc_(gc.impl_->gc(), threadData, gcScheduler), #ifndef CUSTOM_ALLOCATOR objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} #else - alloc_(gc.impl_->gc().heap(), gcScheduler_) {} + alloc_(gc.impl_->gc().heap(), gcScheduler) { + } #endif - GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } GCImpl::ThreadData& gc() noexcept { return gc_; } #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator& alloc() noexcept { return alloc_; } @@ -60,7 +57,6 @@ public: #endif private: - GCSchedulerThreadData gcScheduler_; GCImpl::ThreadData gc_; #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator alloc_; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 1b68ad2978f..83af9af573b 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -27,7 +27,7 @@ public: public: class Impl; - ThreadData(GC& gc, mm::ThreadData& threadData) noexcept; + ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept; ~ThreadData(); Impl& impl() noexcept { return *impl_; } @@ -45,14 +45,13 @@ public: ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; - void OnStoppedForGC() noexcept; void OnSuspendForGC() noexcept; private: std_support::unique_ptr impl_; }; - GC() noexcept; + explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept; ~GC(); Impl& impl() noexcept { return *impl_; } @@ -61,8 +60,6 @@ public: size_t GetTotalHeapObjectsSizeBytes() const noexcept; - gc::GCSchedulerConfig& gcSchedulerConfig() noexcept; - void ClearForTests() noexcept; void StartFinalizerThreadIfNeeded() noexcept; @@ -73,6 +70,9 @@ public: static void processArrayInMark(void* state, ArrayHeader* array) noexcept; static void processFieldInMark(void* state, ObjHeader* field) noexcept; + // TODO: This should be moved into the scheduler. + void Schedule() noexcept; + private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp deleted file mode 100644 index 5dd3875e28a..00000000000 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCScheduler.hpp" - -#include - -#include "CompilerConstants.hpp" -#include "GCSchedulerImpl.hpp" -#include "GlobalData.hpp" -#include "KAssert.h" -#include "Porting.h" -#include "ThreadRegistry.hpp" -#include "ThreadData.hpp" - -using namespace kotlin; - -namespace { - -std_support::unique_ptr MakeGCSchedulerData( - gc::SchedulerType type, gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept { - switch (type) { - case gc::SchedulerType::kDisabled: - RuntimeLogDebug({kTagGC}, "GC scheduler disabled"); - return std_support::make_unique(); - case gc::SchedulerType::kWithTimer: -#ifndef KONAN_NO_THREADS - RuntimeLogDebug({kTagGC}, "Initializing timer-based GC scheduler"); - return std_support::make_unique>(config, std::move(scheduleGC)); -#else - RuntimeFail("GC scheduler with timer is not supported on this platform"); -#endif - case gc::SchedulerType::kOnSafepoints: - RuntimeLogDebug({kTagGC}, "Initializing safe-point-based GC scheduler"); - return std_support::make_unique>(config, std::move(scheduleGC)); - case gc::SchedulerType::kAggressive: - RuntimeLogDebug({kTagGC}, "Initializing aggressive GC scheduler"); - return std_support::make_unique(config, std::move(scheduleGC)); - } -} - -} // namespace - -void gc::GCScheduler::SetScheduleGC(std::function scheduleGC) noexcept { - RuntimeAssert(static_cast(scheduleGC), "scheduleGC cannot be empty"); - RuntimeAssert(!static_cast(scheduleGC_), "scheduleGC must not have been set"); - scheduleGC_ = std::move(scheduleGC); - RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call"); - gcData_ = MakeGCSchedulerData(compiler::getGCSchedulerType(), config_, scheduleGC_); -} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerImpl.hpp deleted file mode 100644 index c08ca065eb2..00000000000 --- a/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerImpl.hpp +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#include "GCScheduler.hpp" - -#include "AppStateTracking.hpp" -#include "Clock.hpp" -#include "GlobalData.hpp" -#include "StackTrace.hpp" -#include "std_support/UnorderedSet.hpp" - -#ifndef KONAN_NO_THREADS -#include "RepeatedTimer.hpp" -#endif - -namespace kotlin::gc::internal { - -class HeapGrowthController { -public: - explicit HeapGrowthController(gc::GCSchedulerConfig& config) noexcept : config_(config) {} - - // Called by the mutators. - void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; } - - // Called by the GC thread. - void OnPerformFullGC() noexcept { allocatedBytes_ = 0; } - - // Called by the GC thread. - void UpdateAliveSetBytes(size_t bytes) noexcept { - lastAliveSetBytes_ = bytes; - - if (config_.autoTune.load()) { - double targetHeapBytes = static_cast(bytes) / config_.targetHeapUtilization; - if (!std::isfinite(targetHeapBytes)) { - // This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything. - return; - } - double minHeapBytes = static_cast(config_.minHeapBytes.load()); - double maxHeapBytes = static_cast(config_.maxHeapBytes.load()); - targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes); - config_.targetHeapBytes = static_cast(targetHeapBytes); - } - } - - // Called by the mutators. - bool NeedsGC() const noexcept { - uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load(); - uint64_t targetHeapBytes = config_.targetHeapBytes; - return currentHeapBytes >= targetHeapBytes; - } - -private: - gc::GCSchedulerConfig& config_; - // Updated by both the mutators and the GC thread. - std::atomic allocatedBytes_ = 0; - // Updated by the GC thread, read by the mutators. - std::atomic lastAliveSetBytes_ = 0; -}; - -template -class RegularIntervalPacer { -public: - using TimePoint = std::chrono::time_point; - - explicit RegularIntervalPacer(gc::GCSchedulerConfig& config) noexcept : config_(config), lastGC_(Clock::now()) {} - - // Called by the mutators or the timer thread. - bool NeedsGC() const noexcept { - auto currentTime = Clock::now(); - return currentTime >= lastGC_.load() + config_.regularGcInterval(); - } - - // Called by the GC thread. - void OnPerformFullGC() noexcept { lastGC_ = Clock::now(); } - -private: - gc::GCSchedulerConfig& config_; - // Updated by the GC thread, read by the mutators or the timer thread. - std::atomic lastGC_; -}; - -template -class SafePointTracker { -public: - using SafePointID = kotlin::StackTrace; - - explicit SafePointTracker(size_t maxSize = 100000) : maxSize_(maxSize) {} - - /** Returns whether the GC must be triggered on the current safe point or not. */ - NO_INLINE bool registerCurrentSafePoint(size_t skipFrames) noexcept { - auto currentSP = SafePointID::current(skipFrames + 1); - - std::unique_lock lock(mutex_); - - // TODO: Consider replacing this naive cleaning with an LRU cache. - if (metSafePoints_.size() >= maxSize()) { - RuntimeLogDebug({kTagGC}, "Clear safe point tracker set since it exceeded maximal size"); - metSafePoints_.clear(); - } - - bool inserted = metSafePoints_.insert(currentSP).second; - return inserted; - } - - size_t maxSize() { return maxSize_; } - - size_t size() { return metSafePoints_.size(); } - -private: - size_t maxSize_; - - // TODO: Consider replacing mutex + global set with thread local sets sychronized on STW. - SpinLock mutex_; - std_support::unordered_set metSafePoints_; -}; - -class GCEmptySchedulerData : public gc::GCSchedulerData { - void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {} - void OnPerformFullGC() noexcept override {} - void UpdateAliveSetBytes(size_t bytes) noexcept override {} -}; - -#ifndef KONAN_NO_THREADS - -template -class GCSchedulerDataWithTimer : public gc::GCSchedulerData { -public: - GCSchedulerDataWithTimer(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept : - config_(config), - appStateTracking_(mm::GlobalData::Instance().appStateTracking()), - heapGrowthController_(config), - regularIntervalPacer_(config), - scheduleGC_(std::move(scheduleGC)), - timer_("GC Timer thread", config_.regularGcInterval(), [this] { - if (appStateTracking_.state() == mm::AppStateTracking::State::kBackground) { - return; - } - if (regularIntervalPacer_.NeedsGC()) { - RuntimeLogInfo({kTagGC}, "Scheduling GC by timer"); - scheduleGC_(); - } - }) {} - - void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { - heapGrowthController_.OnAllocated(threadData.allocatedBytes()); - if (heapGrowthController_.NeedsGC()) { - RuntimeLogInfo({kTagGC}, "Scheduling GC by allocation"); - scheduleGC_(); - } - } - - void OnPerformFullGC() noexcept override { - heapGrowthController_.OnPerformFullGC(); - regularIntervalPacer_.OnPerformFullGC(); - timer_.restart(config_.regularGcInterval()); - } - - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - -private: - gc::GCSchedulerConfig& config_; - mm::AppStateTracking& appStateTracking_; - HeapGrowthController heapGrowthController_; - RegularIntervalPacer regularIntervalPacer_; - std::function scheduleGC_; - RepeatedTimer timer_; -}; - -#endif // !KONAN_NO_THREADS - -template -class GCSchedulerDataOnSafepoints : public gc::GCSchedulerData { -public: - GCSchedulerDataOnSafepoints(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept : - heapGrowthController_(config), regularIntervalPacer_(config), scheduleGC_(std::move(scheduleGC)) {} - - void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { - heapGrowthController_.OnAllocated(threadData.allocatedBytes()); - if (heapGrowthController_.NeedsGC()) { - scheduleGC_(); - } else if (regularIntervalPacer_.NeedsGC()) { - scheduleGC_(); - } - } - - void OnPerformFullGC() noexcept override { - heapGrowthController_.OnPerformFullGC(); - regularIntervalPacer_.OnPerformFullGC(); - } - - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - -private: - HeapGrowthController heapGrowthController_; - RegularIntervalPacer regularIntervalPacer_; - std::function scheduleGC_; -}; - -class GCSchedulerDataAggressive : public gc::GCSchedulerData { -public: - GCSchedulerDataAggressive(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept : - scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) { - // Trigger the slowpath on each safepoint and on each allocation. - // The slowpath will trigger GC if this thread didn't meet this safepoint/allocation site before. - config.threshold = 1; - config.allocationThresholdBytes = 1; - } - - void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { - heapGrowthController_.OnAllocated(threadData.allocatedBytes()); - if (heapGrowthController_.NeedsGC()) { - scheduleGC_(); - } else if (safePointTracker_.registerCurrentSafePoint(1)) { - scheduleGC_(); - } - } - - void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); } - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - -private: - std::function scheduleGC_; - HeapGrowthController heapGrowthController_; - SafePointTracker<> safePointTracker_; -}; - -} // namespace kotlin::gc::internal diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp deleted file mode 100644 index 95bb70c2265..00000000000 --- a/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp +++ /dev/null @@ -1,877 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCScheduler.hpp" - -#include -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -#include "AppStateTrackingTestSupport.hpp" -#include "ClockTestSupport.hpp" -#include "GCSchedulerImpl.hpp" -#include "SingleThreadExecutor.hpp" -#include "TestSupport.hpp" -#include "std_support/Vector.hpp" - -using namespace kotlin; - -using testing::_; - -namespace kotlin { -namespace gc { - -class GCSchedulerThreadDataTestApi : private Pinned { -public: - explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {} - - void OnSafePointRegularImpl(size_t weight) { scheduler_.OnSafePointRegularImpl(weight); } - - void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; } - -private: - GCSchedulerThreadData& scheduler_; -}; - -TEST(GCSchedulerThreadDataTest, RegularSafePoint) { - constexpr size_t kWeight = 2; - constexpr size_t kCount = 10; - constexpr size_t kThreshold = kCount * kWeight; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = 1; - config.threshold = kThreshold; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); - }); - schedulerTestApi.OnSafePointRegularImpl(kWeight); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - schedulerTestApi.OnSafePointRegularImpl(kWeight); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), kWeight); -} - -TEST(GCSchedulerThreadDataTest, AllocationSafePoint) { - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - config.threshold = 1; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - }); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kSize); - EXPECT_THAT(scheduler.safePointsCounter(), 0); -} - -TEST(GCSchedulerThreadDataTest, ResetByGC) { - constexpr size_t kWeight = 2; - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kThreshold = kCount * kWeight; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - config.threshold = kThreshold; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - scheduler.OnStoppedForGC(); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); -} - -TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) { - constexpr size_t kWeight = 2; - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kThreshold = kCount * kWeight; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - config.threshold = kThreshold; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - config.allocationThresholdBytes = kAllocationThreshold - kSize; - config.threshold = kThreshold - kWeight; - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - scheduler.OnSafePointAllocation(kSize); - } - scheduler.OnStoppedForGC(); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); -} - -TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterRegularSafePoint) { - constexpr size_t kWeight = 2; - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kThreshold = kCount * kWeight; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - config.threshold = kThreshold; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - config.allocationThresholdBytes = kAllocationThreshold - kSize; - config.threshold = kThreshold - kWeight; - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); - }); - schedulerTestApi.OnSafePointRegularImpl(kWeight); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); -} - -TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) { - constexpr size_t kWeight = 2; - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kThreshold = kCount * kWeight; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - config.threshold = kThreshold; - GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - config.allocationThresholdBytes = kAllocationThreshold - kSize; - config.threshold = kThreshold - kWeight; - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); - }); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - schedulerTestApi.OnSafePointRegularImpl(kWeight); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - EXPECT_THAT(scheduler.safePointsCounter(), 0); -} - -class MutatorThread : private Pinned { -public: - MutatorThread(GCSchedulerConfig& config, std::function slowPath) : - executor_([&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); }) {} - - std::future Allocate(size_t bytes) { - return executor_.execute([&, bytes] { - auto& context = executor_.context(); - context.threadDataTestApi.SetAllocatedBytes(bytes); - context.slowPath(context.threadData); - }); - } - -private: - struct Context { - GCSchedulerThreadData threadData; - GCSchedulerThreadDataTestApi threadDataTestApi; - std::function slowPath; - - Context(GCSchedulerConfig& config, std::function slowPath) : - threadData(config, [](GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {} - }; - - SingleThreadExecutor executor_; -}; - -template -class GCSchedulerDataTestApi { -public: - explicit GCSchedulerDataTestApi(GCSchedulerConfig& config) : scheduler_(config, scheduleGC_.AsStdFunction()) { - mutators_.reserve(MutatorCount); - for (int i = 0; i < MutatorCount; ++i) { - mutators_.emplace_back(std_support::make_unique( - config, [this](GCSchedulerThreadData& threadData) { scheduler_.UpdateFromThreadData(threadData); })); - } - } - - std::future Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); } - - void OnPerformFullGC() { scheduler_.OnPerformFullGC(); } - - void UpdateAliveSetBytes(size_t bytes) { scheduler_.UpdateAliveSetBytes(bytes); } - - testing::MockFunction& scheduleGC() { return scheduleGC_; } - - template - void advance_time(Duration duration) { - test_support::manual_clock::sleep_for(duration); - } - -private: - std_support::vector> mutators_; - testing::MockFunction scheduleGC_; - GCScheduler scheduler_; -}; - -template -using GCSchedulerDataOnSafepointsTestApi = - GCSchedulerDataTestApi, MutatorCount>; - -template -using GCSchedulerDataWithTimerTestApi = - GCSchedulerDataTestApi, MutatorCount>; - -class GCSchedulerDataTest : public ::testing::Test { -public: - GCSchedulerDataTest() { test_support::manual_clock::reset(); } -}; - -class GCSchedulerDataOnSafePointsTest : public GCSchedulerDataTest {}; -class GCSchedulerDataWithTimerTest : public GCSchedulerDataTest {}; - -TEST_F(GCSchedulerDataOnSafePointsTest, CollectOnTargetHeapReached) { - test_support::manual_clock::reset(); - - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = (mutatorsCount + 1) * 10; - GCSchedulerDataOnSafepointsTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - std_support::vector> futures; - for (int i = 0; i < mutatorsCount; ++i) { - futures.push_back(schedulerTestApi.Allocate(i, 10)); - } - for (auto& future : futures) { - future.get(); - } - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -TEST_F(GCSchedulerDataOnSafePointsTest, CollectOnTimeoutReached) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = std::numeric_limits::max(); - GCSchedulerDataOnSafepointsTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - schedulerTestApi.advance_time(microseconds(5)); - std_support::vector> futures; - for (int i = 0; i < mutatorsCount; ++i) { - futures.push_back(schedulerTestApi.Allocate(i, 0)); - } - for (auto& future : futures) { - future.get(); - } - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.advance_time(microseconds(5)); - schedulerTestApi.Allocate(0, 0).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -TEST_F(GCSchedulerDataOnSafePointsTest, FullTimeoutAfterLastGC) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = 10; - GCSchedulerDataOnSafepointsTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.advance_time(microseconds(5)); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - schedulerTestApi.advance_time(microseconds(5)); - schedulerTestApi.Allocate(0, 0).get(); - // It's now 10 us since the start, but only 5 us since previous collection. - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.advance_time(microseconds(5)); - schedulerTestApi.Allocate(0, 0).get(); - // It's now 10 us since the previous collection. - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -TEST_F(GCSchedulerDataOnSafePointsTest, DoNotTuneTargetHeap) { - constexpr int mutatorsCount = 1; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = 10; - GCSchedulerDataOnSafepointsTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); - - EXPECT_THAT(config.targetHeapBytes.load(), 10); -} - -TEST_F(GCSchedulerDataOnSafePointsTest, TuneTargetHeap) { - constexpr int mutatorsCount = 1; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = true; - config.targetHeapBytes = 10; - config.targetHeapUtilization = 0.5; - config.minHeapBytes = 5; - config.maxHeapBytes = 50; - GCSchedulerDataOnSafepointsTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); - - EXPECT_THAT(config.targetHeapBytes.load(), 20); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 20. - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(20); - - EXPECT_THAT(config.targetHeapBytes.load(), 40); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 60. - schedulerTestApi.Allocate(0, 40).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); - - // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // Keeping total heap of 60. - schedulerTestApi.Allocate(0, 0).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); - - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 0).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - // Dropping to 40 - schedulerTestApi.UpdateAliveSetBytes(40); - - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 50 - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - // Dropping to 1 - schedulerTestApi.UpdateAliveSetBytes(1); - - // But the minimum is set to 5. - EXPECT_THAT(config.targetHeapBytes.load(), 5); -} - -TEST_F(GCSchedulerDataWithTimerTest, CollectOnTargetHeapReached) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = (mutatorsCount + 1) * 10; - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - std_support::vector> futures; - for (int i = 0; i < mutatorsCount; ++i) { - futures.push_back(schedulerTestApi.Allocate(i, 10)); - } - for (auto& future : futures) { - future.get(); - } - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -TEST_F(GCSchedulerDataWithTimerTest, CollectOnTimeoutReached) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = std::numeric_limits::max(); - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - // Wait until the timer is initialized. - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.advance_time(microseconds(10)); - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -TEST_F(GCSchedulerDataWithTimerTest, FullTimeoutAfterLastGC) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = 10; - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - // Wait until the timer is initialized. - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - - schedulerTestApi.advance_time(microseconds(5)); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); - - // pending should restart to be 10us since the previous collection without scheduling another GC. - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); -} - -TEST_F(GCSchedulerDataWithTimerTest, DoNotTuneTargetHeap) { - constexpr int mutatorsCount = 1; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = 10; - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); - - EXPECT_THAT(config.targetHeapBytes.load(), 10); -} - -TEST_F(GCSchedulerDataWithTimerTest, TuneTargetHeap) { - constexpr int mutatorsCount = 1; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = true; - config.targetHeapBytes = 10; - config.targetHeapUtilization = 0.5; - config.minHeapBytes = 5; - config.maxHeapBytes = 50; - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); - - EXPECT_THAT(config.targetHeapBytes.load(), 20); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 20. - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(20); - - EXPECT_THAT(config.targetHeapBytes.load(), 40); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 60. - schedulerTestApi.Allocate(0, 40).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); - - // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // Keeping total heap of 60. - schedulerTestApi.Allocate(0, 0).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); - - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 0).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - // Dropping to 40 - schedulerTestApi.UpdateAliveSetBytes(40); - - EXPECT_THAT(config.targetHeapBytes.load(), 50); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - // For a total heap of 50 - schedulerTestApi.Allocate(0, 10).get(); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - // Dropping to 1 - schedulerTestApi.UpdateAliveSetBytes(1); - - // But the minimum is set to 5. - EXPECT_THAT(config.targetHeapBytes.load(), 5); -} - -TEST_F(GCSchedulerDataWithTimerTest, DoNotCollectOnTimerInBackground) { - constexpr int mutatorsCount = kDefaultThreadCount; - - GCSchedulerConfig config; - config.regularGcIntervalMicroseconds = 10; - config.autoTune = false; - config.targetHeapBytes = std::numeric_limits::max(); - GCSchedulerDataWithTimerTestApi schedulerTestApi(config); - - // TODO: Not a global, please. - mm::AppStateTrackingTestSupport appStateTracking(mm::GlobalData::Instance().appStateTracking()); - - // Wait until the timer is initialized. - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - - // Now go into the background. - ASSERT_THAT(mm::GlobalData::Instance().appStateTracking().state(), mm::AppStateTracking::State::kForeground); - appStateTracking.setState(mm::AppStateTracking::State::kBackground); - - // Timer works in the background, but does nothing. - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); - schedulerTestApi.advance_time(microseconds(10)); - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - - // Now go back into the foreground. - appStateTracking.setState(mm::AppStateTracking::State::kForeground); - - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.advance_time(microseconds(10)); - test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); -} - -// These tests require a stack trace to contain call site addresses but -// on Windows a trace contains function addresses instead. -// So skip these tests on Windows. -#if (__MINGW32__ || __MINGW64__) -#define SKIP_ON_WINDOWS() do { GTEST_SKIP() << "Skip on Windows"; } while(false) -#else -#define SKIP_ON_WINDOWS() do { } while(false) -#endif - -TEST(SafePointTrackerTest, RegisterSafePoints) { - SKIP_ON_WINDOWS(); - []() OPTNONE { - internal::SafePointTracker<> tracker; - - for (size_t i = 0; i < 10; i++) { - bool registered1 = tracker.registerCurrentSafePoint(0); - bool registered2 = tracker.registerCurrentSafePoint(0); - - bool expected = (i == 0); - - EXPECT_THAT(registered1, expected); - EXPECT_THAT(registered2, expected); - } - }(); -} - -template -OPTNONE bool registerCurrentSafePoint(internal::SafePointTracker& tracker) { - return tracker.registerCurrentSafePoint(0); -} - -TEST(SafePointTrackerTest, TrackTopFramesOnly) { - SKIP_ON_WINDOWS(); - []() OPTNONE { - internal::SafePointTracker<16> longTracker; - internal::SafePointTracker<1> shortTracker; - - bool longRegistered1 = registerCurrentSafePoint(longTracker); - bool longRegistered2 = registerCurrentSafePoint(longTracker); - - EXPECT_THAT(longRegistered1, true); - EXPECT_THAT(longRegistered2, true); - - bool shortRegistered1 = registerCurrentSafePoint(shortTracker); - bool shortRegistered2 = registerCurrentSafePoint(shortTracker); - - EXPECT_THAT(shortRegistered1, true); - EXPECT_THAT(shortRegistered2, false); - }(); -} - -TEST(SafePointTrackerTest, CleanOnSizeLimit) { - SKIP_ON_WINDOWS(); - []() OPTNONE { - internal::SafePointTracker<> tracker(2); - - ASSERT_THAT(tracker.size(), 0); - ASSERT_THAT(tracker.maxSize(), 2); - - for (size_t i = 0; i < 3; i++) { - bool registered1 = tracker.registerCurrentSafePoint(0); - - EXPECT_THAT(registered1, true); - EXPECT_THAT(tracker.size(), 1); - - bool registered2 = tracker.registerCurrentSafePoint(0); - - EXPECT_THAT(registered2, true); - EXPECT_THAT(tracker.size(), 2); - } - }(); -} - -TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) { - SKIP_ON_WINDOWS(); - []() OPTNONE { - testing::MockFunction scheduleGC; - - GCSchedulerConfig config; - gc::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); - ASSERT_EQ(config.threshold, 1); - - GCSchedulerThreadData threadSchedulerData(config, [](GCSchedulerThreadData&){}); - - EXPECT_CALL(scheduleGC, Call()).Times(1); - for (int i = 0; i < 10; i++) { - scheduler.UpdateFromThreadData(threadSchedulerData); - } - testing::Mock::VerifyAndClearExpectations(&scheduleGC); - - EXPECT_CALL(scheduleGC, Call()).Times(1); - scheduler.UpdateFromThreadData(threadSchedulerData); - testing::Mock::VerifyAndClearExpectations(&scheduleGC); - }(); -} - -TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { - SKIP_ON_WINDOWS(); - []() OPTNONE { - testing::MockFunction scheduleGC; - - GCSchedulerConfig config; - gc::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); - GCSchedulerThreadData threadSchedulerData(config, [&scheduler](GCSchedulerThreadData& data){ - scheduler.UpdateFromThreadData(data); - }); - - ASSERT_EQ(config.allocationThresholdBytes, 1); - - config.autoTune = false; - config.targetHeapBytes = 10; - - int i = 0; - // We trigger GC on the first iteration, when the unique allocation point is faced, - // and on the last iteration when target heap size is reached. - EXPECT_CALL(scheduleGC, Call()) - .WillOnce([&i]() { EXPECT_THAT(i, 0); }) - .WillOnce([&i]() { EXPECT_THAT(i, 9); }); - - for (; i < 10; i++) { - threadSchedulerData.OnSafePointAllocation(1); - } - testing::Mock::VerifyAndClearExpectations(&scheduleGC); - }(); -} - -} // namespace gc -} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index fd9f1f66d7a..9595d8694ab 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -169,7 +169,7 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T template void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { auto handle = gcHandle.collectThreadRoots(thread); - thread.gc().OnStoppedForGC(); + thread.gcScheduler().OnStoppedForGC(); // TODO: Remove useless mm::ThreadRootSet abstraction. for (auto value : mm::ThreadRootSet(thread)) { if (internal::collectRoot(markQueue, value.object)) { diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index f60b61263e4..6ad027b22f4 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -12,7 +12,8 @@ using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData&, mm::ThreadData& threadData) noexcept : + impl_(std_support::make_unique(gc, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; @@ -52,13 +53,9 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); } -void gc::GC::ThreadData::OnStoppedForGC() noexcept { - impl_->gcScheduler().OnStoppedForGC(); -} - void gc::GC::ThreadData::OnSuspendForGC() noexcept { } -gc::GC::GC() noexcept : impl_(std_support::make_unique()) {} +gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique()) {} gc::GC::~GC() = default; @@ -71,10 +68,6 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } -gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { - return impl_->gcScheduler().config(); -} - void gc::GC::ClearForTests() noexcept { impl_->objectFactory().ClearForTests(); GCHandle::ClearForTests(); @@ -96,3 +89,5 @@ ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) n // static ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {} + +void gc::GC::Schedule() noexcept {} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 28b07466c6b..83c3d2494ce 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -8,6 +8,7 @@ #include "GC.hpp" #include "NoOpGC.hpp" +#include "ObjectFactory.hpp" namespace kotlin { namespace gc { @@ -16,31 +17,25 @@ using GCImpl = NoOpGC; class GC::Impl : private Pinned { public: - Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + Impl() noexcept = default; mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } - GCScheduler& gcScheduler() noexcept { return gcScheduler_; } GCImpl& gc() noexcept { return gc_; } private: mm::ObjectFactory objectFactory_; - GCScheduler gcScheduler_; GCImpl gc_; }; class GC::ThreadData::Impl : private Pinned { public: Impl(GC& gc, mm::ThreadData& threadData) noexcept : - gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), - gc_(gc.impl_->gc(), threadData, gcScheduler_), objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} - GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } GCImpl::ThreadData& gc() noexcept { return gc_; } mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } private: - GCSchedulerThreadData gcScheduler_; GCImpl::ThreadData gc_; mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 7a514c3e782..dba87b7137a 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -9,10 +9,8 @@ #include #include "Allocator.hpp" -#include "GCScheduler.hpp" -#include "ObjectFactory.hpp" +#include "Logging.hpp" #include "Utils.hpp" -#include "Types.h" namespace kotlin { @@ -34,7 +32,7 @@ public: public: using ObjectData = NoOpGC::ObjectData; - ThreadData(NoOpGC& gc, mm::ThreadData& threadData, GCSchedulerThreadData&) noexcept {} + ThreadData() noexcept {} ~ThreadData() = default; void SafePointFunctionPrologue() noexcept {} @@ -52,15 +50,10 @@ public: private: }; - NoOpGC(mm::ObjectFactory&, GCScheduler&) noexcept { - RuntimeLogDebug({kTagGC}, "No-op GC initialized"); - } + NoOpGC() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); } ~NoOpGC() = default; - GCScheduler& scheduler() noexcept { return scheduler_; } - private: - GCScheduler scheduler_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 4ef5106c2a2..63207dda329 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -13,25 +13,17 @@ using namespace kotlin; -namespace { - -ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept { - threadData.impl().gcScheduler().OnSafePointRegular(weight); - mm::SuspendIfRequested(); -} - -} // namespace - -gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : + impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept { - SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight); + mm::SuspendIfRequested(); } ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight); + mm::SuspendIfRequested(); } void gc::GC::ThreadData::Schedule() noexcept { @@ -62,13 +54,9 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); } -void gc::GC::ThreadData::OnStoppedForGC() noexcept { - impl_->gcScheduler().OnStoppedForGC(); -} - void gc::GC::ThreadData::OnSuspendForGC() noexcept { } -gc::GC::GC() noexcept : impl_(std_support::make_unique()) {} +gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} gc::GC::~GC() = default; @@ -81,10 +69,6 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } -gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { - return impl_->gcScheduler().config(); -} - void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); impl_->objectFactory().ClearForTests(); @@ -117,3 +101,7 @@ ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) n ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { gc::internal::processFieldInMark(state, field); } + +void gc::GC::Schedule() noexcept { + impl_->gc().Schedule(); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index 8cf02282d38..8027028891a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -16,31 +16,25 @@ using GCImpl = SameThreadMarkAndSweep; class GC::Impl : private Pinned { public: - Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {} mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } - GCScheduler& gcScheduler() noexcept { return gcScheduler_; } GCImpl& gc() noexcept { return gc_; } private: mm::ObjectFactory objectFactory_; - GCScheduler gcScheduler_; GCImpl gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : - gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), - gc_(gc.impl_->gc(), threadData, gcScheduler_), - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} + Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : + gc_(gc.impl_->gc(), threadData, gcScheduler), objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} - GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } GCImpl::ThreadData& gc() noexcept { return gc_; } mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } private: - GCSchedulerThreadData gcScheduler_; GCImpl::ThreadData gc_; mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 66c9db6e67f..cefda3da458 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -84,17 +84,11 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { } gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : + mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { GCHandle::getByEpoch(epoch).finalizersDone(); state_.finalized(epoch); }) { - gcScheduler_.SetScheduleGC([this]() NO_INLINE { - RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); - // This call acquires a lock, so we need to ensure that we're in the safe state. - NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); - state_.schedule(); - }); gcThread_ = ScopedThread(ScopedThread::attributes().name("GC thread"), [this] { while (true) { auto epoch = state_.waitScheduled(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index a9d5b116b5c..04c08ce1454 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -70,7 +70,7 @@ public: using ObjectData = SameThreadMarkAndSweep::ObjectData; using Allocator = AllocatorWithGC; - ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : + ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : gc_(gc), gcScheduler_(gcScheduler) {} ~ThreadData() = default; @@ -87,7 +87,7 @@ public: private: SameThreadMarkAndSweep& gc_; - GCSchedulerThreadData& gcScheduler_; + gcScheduler::GCSchedulerThreadData& gcScheduler_; }; using Allocator = ThreadData::Allocator; @@ -95,18 +95,20 @@ public: using FinalizerQueue = mm::ObjectFactory::FinalizerQueue; using FinalizerQueueTraits = mm::ObjectFactory::FinalizerQueueTraits; - SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept; + SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept; ~SameThreadMarkAndSweep(); void StartFinalizerThreadIfNeeded() noexcept; void StopFinalizerThreadIfRunning() noexcept; bool FinalizersThreadIsRunning() noexcept; + void Schedule() noexcept { state_.schedule(); } + private: void PerformFullGC(int64_t epoch) noexcept; mm::ObjectFactory& objectFactory_; - GCScheduler& gcScheduler_; + gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; ScopedThread gcThread_; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 9f90da89fa7..a2ecc10a2d1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -725,7 +725,8 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) { std_support::vector> gcFutures(kDefaultThreadCount); - gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); + gcFutures[0] = mutators[0].Execute( + [](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); // Spin until thread suspension is requested. while (!mm::IsThreadSuspensionRequested()) { @@ -857,7 +858,8 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRe } std_support::vector> gcFutures(kDefaultThreadCount); - gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); + gcFutures[0] = mutators[0].Execute( + [](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); // Spin until thread suspension is requested. while (!mm::IsThreadSuspensionRequested()) { @@ -922,7 +924,8 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { std_support::vector> gcFutures(kDefaultThreadCount); - gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); + gcFutures[0] = mutators[0].Execute( + [](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); // Spin until thread suspension is requested. while (!mm::IsThreadSuspensionRequested()) { @@ -1035,7 +1038,8 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { std_support::vector> gcFutures(kDefaultThreadCount); - gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); + gcFutures[0] = mutators[0].Execute( + [](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); }); // Spin until thread suspension is requested. while (!mm::IsThreadSuspensionRequested()) { diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp new file mode 100644 index 00000000000..1f53a9eb5e5 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerImpl.hpp" + +#include "GlobalData.hpp" +#include "Memory.h" +#include "Logging.hpp" +#include "Porting.h" + +using namespace kotlin; + +void gcScheduler::GCSchedulerThreadData::OnSafePointRegular(size_t weight) noexcept {} + +gcScheduler::GCScheduler::GCScheduler() noexcept : + gcData_(std_support::make_unique>(config_, []() noexcept { + // This call acquires a lock, so we need to ensure that we're in the safe state. + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().Schedule(); + })) {} diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp new file mode 100644 index 00000000000..d9e544c6e35 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -0,0 +1,66 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GCScheduler.hpp" + +#include "AppStateTracking.hpp" +#include "GCSchedulerConfig.hpp" +#include "GlobalData.hpp" +#include "HeapGrowthController.hpp" +#include "Logging.hpp" +#include "RegularIntervalPacer.hpp" +#include "RepeatedTimer.hpp" + +namespace kotlin::gcScheduler::internal { + +template +class GCSchedulerDataAdaptive : public GCSchedulerData { +public: + GCSchedulerDataAdaptive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : + config_(config), + scheduleGC_(std::move(scheduleGC)), + appStateTracking_(mm::GlobalData::Instance().appStateTracking()), + heapGrowthController_(config), + regularIntervalPacer_(config), + timer_("GC Timer thread", config_.regularGcInterval(), [this] { + if (appStateTracking_.state() == mm::AppStateTracking::State::kBackground) { + return; + } + if (regularIntervalPacer_.NeedsGC()) { + RuntimeLogDebug({kTagGC}, "Scheduling GC by timer"); + scheduleGC_(); + } + }) { + RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized"); + } + + void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override { + heapGrowthController_.OnAllocated(threadData.allocatedBytes()); + if (heapGrowthController_.NeedsGC()) { + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + scheduleGC_(); + } + } + + void OnPerformFullGC() noexcept override { + heapGrowthController_.OnPerformFullGC(); + regularIntervalPacer_.OnPerformFullGC(); + timer_.restart(config_.regularGcInterval()); + } + + void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } + +private: + GCSchedulerConfig& config_; + std::function scheduleGC_; + mm::AppStateTracking& appStateTracking_; + HeapGrowthController heapGrowthController_; + RegularIntervalPacer regularIntervalPacer_; + RepeatedTimer timer_; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp new file mode 100644 index 00000000000..df1ffa531b6 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp @@ -0,0 +1,288 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerImpl.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "AppStateTrackingTestSupport.hpp" +#include "ClockTestSupport.hpp" +#include "GCSchedulerTestSupport.hpp" +#include "SingleThreadExecutor.hpp" +#include "TestSupport.hpp" +#include "std_support/Vector.hpp" + +using namespace kotlin; + +namespace { + +class MutatorThread : private Pinned { +public: + MutatorThread(gcScheduler::GCSchedulerConfig& config, std::function slowPath) : + executor_([&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); }) {} + + std::future Allocate(size_t bytes) { + return executor_.execute([&, bytes] { + auto& context = executor_.context(); + context.threadDataTestApi.SetAllocatedBytes(bytes); + context.slowPath(context.threadData); + }); + } + +private: + struct Context { + gcScheduler::GCSchedulerThreadData threadData; + gcScheduler::test_support::GCSchedulerThreadDataTestApi threadDataTestApi; + std::function slowPath; + + Context(gcScheduler::GCSchedulerConfig& config, std::function slowPath) : + threadData(config, [](gcScheduler::GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {} + }; + + SingleThreadExecutor executor_; +}; + +template +class GCSchedulerDataTestApi { +public: + explicit GCSchedulerDataTestApi(gcScheduler::GCSchedulerConfig& config) : scheduler_(config, scheduleGC_.AsStdFunction()) { + mutators_.reserve(MutatorCount); + for (int i = 0; i < MutatorCount; ++i) { + mutators_.emplace_back(std_support::make_unique( + config, [this](gcScheduler::GCSchedulerThreadData& threadData) { scheduler_.UpdateFromThreadData(threadData); })); + } + } + + std::future Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); } + + void OnPerformFullGC() { scheduler_.OnPerformFullGC(); } + + void UpdateAliveSetBytes(size_t bytes) { scheduler_.UpdateAliveSetBytes(bytes); } + + testing::MockFunction& scheduleGC() { return scheduleGC_; } + + template + void advance_time(Duration duration) { + test_support::manual_clock::sleep_for(duration); + } + +private: + std_support::vector> mutators_; + testing::MockFunction scheduleGC_; + gcScheduler::internal::GCSchedulerDataAdaptive scheduler_; +}; + +} // namespace + +class AdaptiveSchedulerTest : public ::testing::Test { +public: + AdaptiveSchedulerTest() { test_support::manual_clock::reset(); } +}; + +TEST_F(AdaptiveSchedulerTest, CollectOnTargetHeapReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = (mutatorsCount + 1) * 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + std_support::vector> futures; + for (int i = 0; i < mutatorsCount; ++i) { + futures.push_back(schedulerTestApi.Allocate(i, 10)); + } + for (auto& future : futures) { + future.get(); + } + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = std::numeric_limits::max(); + GCSchedulerDataTestApi schedulerTestApi(config); + + // Wait until the timer is initialized. + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(microseconds(10)); + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) { + constexpr int mutatorsCount = kDefaultThreadCount; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + // Wait until the timer is initialized. + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + + schedulerTestApi.advance_time(microseconds(5)); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + // pending should restart to be 10us since the previous collection without scheduling another GC. + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); +} + +TEST_F(AdaptiveSchedulerTest, DoNotTuneTargetHeap) { + constexpr int mutatorsCount = 1; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 10); +} + +TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { + constexpr int mutatorsCount = 1; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = true; + config.targetHeapBytes = 10; + config.targetHeapUtilization = 0.5; + config.minHeapBytes = 5; + config.maxHeapBytes = 50; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 20); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 20. + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(20); + + EXPECT_THAT(config.targetHeapBytes.load(), 40); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 60. + schedulerTestApi.Allocate(0, 40).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // Keeping total heap of 60. + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 40 + schedulerTestApi.UpdateAliveSetBytes(40); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 50 + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 1 + schedulerTestApi.UpdateAliveSetBytes(1); + + // But the minimum is set to 5. + EXPECT_THAT(config.targetHeapBytes.load(), 5); +} + +TEST_F(AdaptiveSchedulerTest, DoNotCollectOnTimerInBackground) { + constexpr int mutatorsCount = kDefaultThreadCount; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = std::numeric_limits::max(); + GCSchedulerDataTestApi schedulerTestApi(config); + + // TODO: Not a global, please. + mm::AppStateTrackingTestSupport appStateTracking(mm::GlobalData::Instance().appStateTracking()); + + // Wait until the timer is initialized. + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + + // Now go into the background. + ASSERT_THAT(mm::GlobalData::Instance().appStateTracking().state(), mm::AppStateTracking::State::kForeground); + appStateTracking.setState(mm::AppStateTracking::State::kBackground); + + // Timer works in the background, but does nothing. + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(microseconds(10)); + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + // Now go back into the foreground. + appStateTracking.setState(mm::AppStateTracking::State::kForeground); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(microseconds(10)); + test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp new file mode 100644 index 00000000000..0b95069d768 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerImpl.hpp" + +#include "GlobalData.hpp" +#include "Memory.h" +#include "Logging.hpp" +#include "Porting.h" + +using namespace kotlin; + +void gcScheduler::GCSchedulerThreadData::OnSafePointRegular(size_t weight) noexcept { + OnSafePointRegularImpl(weight); +} + +gcScheduler::GCScheduler::GCScheduler() noexcept : + gcData_(std_support::make_unique(config_, []() noexcept { + // This call acquires a lock, so we need to ensure that we're in the safe state. + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().Schedule(); + })) {} diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp new file mode 100644 index 00000000000..7c0ad992312 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GCScheduler.hpp" + +#include + +#include "GCSchedulerConfig.hpp" +#include "HeapGrowthController.hpp" +#include "Logging.hpp" +#include "SafePointTracker.hpp" + +namespace kotlin::gcScheduler::internal { + +class GCSchedulerDataAggressive : public GCSchedulerData { +public: + GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : + scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) { + // Trigger the slowpath on each safepoint and on each allocation. + // The slowpath will trigger GC if this thread didn't meet this safepoint/allocation site before. + config.threshold = 1; + config.allocationThresholdBytes = 1; + RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized"); + } + + void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override { + heapGrowthController_.OnAllocated(threadData.allocatedBytes()); + if (heapGrowthController_.NeedsGC()) { + // Still checking allocations: with a long running loop all safepoints + // might be "met", so that's the only trigger to not run out of memory. + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + scheduleGC_(); + } else if (safePointTracker_.registerCurrentSafePoint(1)) { + RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint"); + scheduleGC_(); + } + } + + void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); } + void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } + +private: + std::function scheduleGC_; + HeapGrowthController heapGrowthController_; + SafePointTracker<> safePointTracker_; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp new file mode 100644 index 00000000000..7f7c17e34b5 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp @@ -0,0 +1,75 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerImpl.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace kotlin; + +// These tests require a stack trace to contain call site addresses but +// on Windows a trace contains function addresses instead. +// So skip these tests on Windows. +#if (__MINGW32__ || __MINGW64__) +#define SKIP_ON_WINDOWS() \ + do { \ + GTEST_SKIP() << "Skip on Windows"; \ + } while (false) +#else +#define SKIP_ON_WINDOWS() \ + do { \ + } while (false) +#endif + +TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) { + SKIP_ON_WINDOWS(); + []() OPTNONE { + testing::MockFunction scheduleGC; + + gcScheduler::GCSchedulerConfig config; + gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); + ASSERT_EQ(config.threshold, 1); + + gcScheduler::GCSchedulerThreadData threadSchedulerData(config, [](gcScheduler::GCSchedulerThreadData&) {}); + + EXPECT_CALL(scheduleGC, Call()).Times(1); + for (int i = 0; i < 10; i++) { + scheduler.UpdateFromThreadData(threadSchedulerData); + } + testing::Mock::VerifyAndClearExpectations(&scheduleGC); + + EXPECT_CALL(scheduleGC, Call()).Times(1); + scheduler.UpdateFromThreadData(threadSchedulerData); + testing::Mock::VerifyAndClearExpectations(&scheduleGC); + }(); +} + +TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { + SKIP_ON_WINDOWS(); + []() OPTNONE { + testing::MockFunction scheduleGC; + + gcScheduler::GCSchedulerConfig config; + gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); + gcScheduler::GCSchedulerThreadData threadSchedulerData( + config, [&scheduler](gcScheduler::GCSchedulerThreadData& data) { scheduler.UpdateFromThreadData(data); }); + + ASSERT_EQ(config.allocationThresholdBytes, 1); + + config.autoTune = false; + config.targetHeapBytes = 10; + + int i = 0; + // We trigger GC on the first iteration, when the unique allocation point is faced, + // and on the last iteration when target heap size is reached. + EXPECT_CALL(scheduleGC, Call()).WillOnce([&i]() { EXPECT_THAT(i, 0); }).WillOnce([&i]() { EXPECT_THAT(i, 9); }); + + for (; i < 10; i++) { + threadSchedulerData.OnSafePointAllocation(1); + } + testing::Mock::VerifyAndClearExpectations(&scheduleGC); + }(); +} diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTracker.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTracker.hpp new file mode 100644 index 00000000000..6420af306da --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTracker.hpp @@ -0,0 +1,53 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include + +#include "KAssert.h" +#include "Logging.hpp" +#include "Mutex.hpp" +#include "StackTrace.hpp" +#include "std_support/UnorderedSet.hpp" + +namespace kotlin::gcScheduler::internal { + +template +class SafePointTracker { +public: + using SafePointID = kotlin::StackTrace; + + explicit SafePointTracker(size_t maxSize = 100000) : maxSize_(maxSize) {} + + /** Returns whether the GC must be triggered on the current safe point or not. */ + NO_INLINE bool registerCurrentSafePoint(size_t skipFrames) noexcept { + auto currentSP = SafePointID::current(skipFrames + 1); + + std::unique_lock lock(mutex_); + + // TODO: Consider replacing this naive cleaning with an LRU cache. + if (metSafePoints_.size() >= maxSize()) { + RuntimeLogDebug({kTagGC}, "Clear safe point tracker set since it exceeded maximal size"); + metSafePoints_.clear(); + } + + bool inserted = metSafePoints_.insert(currentSP).second; + return inserted; + } + + size_t maxSize() { return maxSize_; } + + size_t size() { return metSafePoints_.size(); } + +private: + size_t maxSize_; + + // TODO: Consider replacing mutex + global set with thread local sets sychronized on STW. + SpinLock mutex_; + std_support::unordered_set metSafePoints_; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTrackerTest.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTrackerTest.cpp new file mode 100644 index 00000000000..fe72a11f42c --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/SafePointTrackerTest.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "SafePointTracker.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace kotlin; + +// These tests require a stack trace to contain call site addresses but +// on Windows a trace contains function addresses instead. +// So skip these tests on Windows. +#if (__MINGW32__ || __MINGW64__) +#define SKIP_ON_WINDOWS() \ + do { \ + GTEST_SKIP() << "Skip on Windows"; \ + } while (false) +#else +#define SKIP_ON_WINDOWS() \ + do { \ + } while (false) +#endif + +TEST(SafePointTrackerTest, RegisterSafePoints) { + SKIP_ON_WINDOWS(); + []() OPTNONE { + gcScheduler::internal::SafePointTracker<> tracker; + + for (size_t i = 0; i < 10; i++) { + bool registered1 = tracker.registerCurrentSafePoint(0); + bool registered2 = tracker.registerCurrentSafePoint(0); + + bool expected = (i == 0); + + EXPECT_THAT(registered1, expected); + EXPECT_THAT(registered2, expected); + } + }(); +} + +template +OPTNONE bool registerCurrentSafePoint(gcScheduler::internal::SafePointTracker& tracker) { + return tracker.registerCurrentSafePoint(0); +} + +TEST(SafePointTrackerTest, TrackTopFramesOnly) { + SKIP_ON_WINDOWS(); + []() OPTNONE { + gcScheduler::internal::SafePointTracker<16> longTracker; + gcScheduler::internal::SafePointTracker<1> shortTracker; + + bool longRegistered1 = registerCurrentSafePoint(longTracker); + bool longRegistered2 = registerCurrentSafePoint(longTracker); + + EXPECT_THAT(longRegistered1, true); + EXPECT_THAT(longRegistered2, true); + + bool shortRegistered1 = registerCurrentSafePoint(shortTracker); + bool shortRegistered2 = registerCurrentSafePoint(shortTracker); + + EXPECT_THAT(shortRegistered1, true); + EXPECT_THAT(shortRegistered2, false); + }(); +} + +TEST(SafePointTrackerTest, CleanOnSizeLimit) { + SKIP_ON_WINDOWS(); + []() OPTNONE { + gcScheduler::internal::SafePointTracker<> tracker(2); + + ASSERT_THAT(tracker.size(), 0); + ASSERT_THAT(tracker.maxSize(), 2); + + for (size_t i = 0; i < 3; i++) { + bool registered1 = tracker.registerCurrentSafePoint(0); + + EXPECT_THAT(registered1, true); + EXPECT_THAT(tracker.size(), 1); + + bool registered2 = tracker.registerCurrentSafePoint(0); + + EXPECT_THAT(registered2, true); + EXPECT_THAT(tracker.size(), 2); + } + }(); +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.cpp new file mode 100644 index 00000000000..8107ee19351 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.cpp @@ -0,0 +1,6 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCScheduler.hpp" diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp similarity index 53% rename from kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp rename to kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp index 993446cc4aa..8cc12d5f6ca 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp @@ -3,50 +3,22 @@ * that can be found in the LICENSE file. */ -#ifndef RUNTIME_GC_COMMON_GC_SCHEDULER_H -#define RUNTIME_GC_COMMON_GC_SCHEDULER_H +#pragma once -#include -#include -#include #include #include #include -#include "CompilerConstants.hpp" -#include "Logging.hpp" -#include "Types.h" +#include "GCSchedulerConfig.hpp" +#include "KAssert.h" #include "Utils.hpp" #include "std_support/Memory.hpp" -namespace kotlin { -namespace gc { +namespace kotlin::gcScheduler { -using SchedulerType = compiler::GCSchedulerType; - -// NOTE: When changing default values, reflect them in GC.kt as well. -struct GCSchedulerConfig { - // Only used when useGCTimer() is false. How many regular safepoints will trigger slow path. - std::atomic threshold = 100000; - // How many object bytes a thread must allocate to trigger slow path. - std::atomic allocationThresholdBytes = 10 * 1024; - std::atomic autoTune = true; - // The target interval between collections when Kotlin code is idle. GC will be triggered - // by timer no sooner than this value and no later than twice this value since the previous collection. - std::atomic regularGcIntervalMicroseconds = 10 * 1000 * 1000; - // How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true. - std::atomic targetHeapBytes = 1024 * 1024; - // The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection - // N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped - // between minHeapBytes and maxHeapBytes. - std::atomic targetHeapUtilization = 0.5; - // The minimum value of targetHeapBytes for autoTune = true - std::atomic minHeapBytes = 1024 * 1024; - // The maximum value of targetHeapBytes for autoTune = true - std::atomic maxHeapBytes = std::numeric_limits::max(); - - std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); } -}; +namespace test_support { +class GCSchedulerThreadDataTestApi; +} class GCSchedulerThreadData; @@ -79,17 +51,7 @@ public: } // Should be called on encountering a safepoint. - void OnSafePointRegular(size_t weight) noexcept { - // TODO: This is a weird design. Consider replacing switch+virtual functions with pimpl+separate compilation. - switch (compiler::getGCSchedulerType()) { - case compiler::GCSchedulerType::kOnSafepoints: - case compiler::GCSchedulerType::kAggressive: - OnSafePointRegularImpl(weight); - return; - default: - return; - } - } + void OnSafePointRegular(size_t weight) noexcept; // Should be called on encountering a safepoint placed by the allocator. // TODO: Should this even be a safepoint (i.e. a place, where we suspend)? @@ -108,7 +70,7 @@ public: size_t safePointsCounter() const noexcept { return safePointsCounter_; } private: - friend class GCSchedulerThreadDataTestApi; + friend class test_support::GCSchedulerThreadDataTestApi; void OnSafePointRegularImpl(size_t weight) noexcept { safePointsCounter_ += weight; @@ -142,17 +104,10 @@ private: class GCScheduler : private Pinned { public: - GCScheduler() noexcept = default; + GCScheduler() noexcept; GCSchedulerConfig& config() noexcept { return config_; } - // Only valid after `SetScheduleGC` is called. - GCSchedulerData& gcData() noexcept { - RuntimeAssert(gcData_ != nullptr, "Cannot be called before SetScheduleGC"); - return *gcData_; - } - - // Can only be called once. - void SetScheduleGC(std::function scheduleGC) noexcept; + GCSchedulerData& gcData() noexcept { return *gcData_; } GCSchedulerThreadData NewThreadData() noexcept { return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->UpdateFromThreadData(threadData); }); @@ -161,10 +116,6 @@ public: private: GCSchedulerConfig config_; std_support::unique_ptr gcData_; - std::function scheduleGC_; }; -} // namespace gc -} // namespace kotlin - -#endif // RUNTIME_GC_COMMON_GC_SCHEDULER_H +} // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp new file mode 100644 index 00000000000..4772a634287 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include +#include + +namespace kotlin::gcScheduler { + +// NOTE: When changing default values, reflect them in GC.kt as well. +struct GCSchedulerConfig { + // Only used when useGCTimer() is false. How many regular safepoints will trigger slow path. + std::atomic threshold = 100000; + // How many object bytes a thread must allocate to trigger slow path. + std::atomic allocationThresholdBytes = 10 * 1024; + std::atomic autoTune = true; + // The target interval between collections when Kotlin code is idle. GC will be triggered + // by timer no sooner than this value and no later than twice this value since the previous collection. + std::atomic regularGcIntervalMicroseconds = 10 * 1000 * 1000; + // How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true. + std::atomic targetHeapBytes = 1024 * 1024; + // The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection + // N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped + // between minHeapBytes and maxHeapBytes. + std::atomic targetHeapUtilization = 0.5; + // The minimum value of targetHeapBytes for autoTune = true + std::atomic minHeapBytes = 1024 * 1024; + // The maximum value of targetHeapBytes for autoTune = true + std::atomic maxHeapBytes = std::numeric_limits::max(); + + std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); } +}; + +} // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp new file mode 100644 index 00000000000..66aeb20a818 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp @@ -0,0 +1,263 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCScheduler.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "GCSchedulerTestSupport.hpp" + +using namespace kotlin; + +using testing::_; + +TEST(GCSchedulerThreadDataTest, RegularSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = 1; + config.threshold = kThreshold; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); + }); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kWeight); +} + +TEST(GCSchedulerThreadDataTest, AllocationSafePoint) { + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = 1; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + }); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kSize); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, ResetByGC) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + scheduler.OnStoppedForGC(); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + scheduler.OnStoppedForGC(); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterRegularSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); + }); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + gcScheduler::GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); + }); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp new file mode 100644 index 00000000000..255d187b666 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp @@ -0,0 +1,6 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerTestSupport.hpp" diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp new file mode 100644 index 00000000000..33f545f4b21 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GCScheduler.hpp" + +namespace kotlin::gcScheduler::test_support { + +class GCSchedulerThreadDataTestApi : private Pinned { +public: + explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {} + + void OnSafePointRegularImpl(size_t weight) { scheduler_.OnSafePointRegularImpl(weight); } + + void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; } + +private: + GCSchedulerThreadData& scheduler_; +}; + +} // namespace kotlin::gcScheduler::test_support diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp new file mode 100644 index 00000000000..259a13d39e0 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp @@ -0,0 +1,60 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "GCSchedulerConfig.hpp" + +namespace kotlin::gcScheduler::internal { + +class HeapGrowthController { +public: + explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : config_(config) {} + + // Called by the mutators. + void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; } + + // Called by the GC thread. + void OnPerformFullGC() noexcept { allocatedBytes_ = 0; } + + // Called by the GC thread. + void UpdateAliveSetBytes(size_t bytes) noexcept { + lastAliveSetBytes_ = bytes; + + if (config_.autoTune.load()) { + double targetHeapBytes = static_cast(bytes) / config_.targetHeapUtilization; + if (!std::isfinite(targetHeapBytes)) { + // This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything. + return; + } + double minHeapBytes = static_cast(config_.minHeapBytes.load()); + double maxHeapBytes = static_cast(config_.maxHeapBytes.load()); + targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes); + config_.targetHeapBytes = static_cast(targetHeapBytes); + } + } + + // Called by the mutators. + bool NeedsGC() const noexcept { + uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load(); + uint64_t targetHeapBytes = config_.targetHeapBytes; + return currentHeapBytes >= targetHeapBytes; + } + +private: + GCSchedulerConfig& config_; + // Updated by both the mutators and the GC thread. + std::atomic allocatedBytes_ = 0; + // Updated by the GC thread, read by the mutators. + std::atomic lastAliveSetBytes_ = 0; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/RegularIntervalPacer.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/RegularIntervalPacer.hpp new file mode 100644 index 00000000000..bec45da453e --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/RegularIntervalPacer.hpp @@ -0,0 +1,36 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include + +#include "GCSchedulerConfig.hpp" + +namespace kotlin::gcScheduler::internal { + +template +class RegularIntervalPacer { +public: + using TimePoint = std::chrono::time_point; + + explicit RegularIntervalPacer(GCSchedulerConfig& config) noexcept : config_(config), lastGC_(Clock::now()) {} + + // Called by the mutators or the timer thread. + bool NeedsGC() const noexcept { + auto currentTime = Clock::now(); + return currentTime >= lastGC_.load() + config_.regularGcInterval(); + } + + // Called by the GC thread. + void OnPerformFullGC() noexcept { lastGC_ = Clock::now(); } + +private: + GCSchedulerConfig& config_; + // Updated by the GC thread, read by the mutators or the timer thread. + std::atomic lastGC_; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp new file mode 100644 index 00000000000..4dac174fef7 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerImpl.hpp" + +using namespace kotlin; + +void gcScheduler::GCSchedulerThreadData::OnSafePointRegular(size_t weight) noexcept {} + +gcScheduler::GCScheduler::GCScheduler() noexcept : gcData_(std_support::make_unique()) {} diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp new file mode 100644 index 00000000000..c1bf97ce2ea --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GCScheduler.hpp" + +#include "Logging.hpp" + +namespace kotlin::gcScheduler::internal { + +class GCSchedulerDataManual : public GCSchedulerData { +public: + GCSchedulerDataManual() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); } + + void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override {} + void OnPerformFullGC() noexcept override {} + void UpdateAliveSetBytes(size_t bytes) noexcept override {} +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp index 1dfaa4008aa..1edfa4b37f3 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp @@ -37,7 +37,6 @@ extern "C" const int32_t Kotlin_needDebugInfo; extern "C" const int32_t Kotlin_runtimeAssertsMode; extern "C" const int32_t Kotlin_disableMmap; extern "C" const char* const Kotlin_runtimeLogs; -extern "C" const int32_t Kotlin_gcSchedulerType; extern "C" const int32_t Kotlin_freezingEnabled; extern "C" const int32_t Kotlin_freezingChecksEnabled; @@ -65,14 +64,6 @@ enum class WorkerExceptionHandling : int32_t { kUseHook = 1, }; -// Must match GCSchedulerType in GCSchedulerType.kt -enum class GCSchedulerType { - kDisabled = 0, - kWithTimer = 1, - kOnSafepoints = 2, - kAggressive = 3, -}; - // Must match AppStateTracking in AppStateTracking.kt enum class AppStateTracking { kDisabled = 0, @@ -107,11 +98,6 @@ ALWAYS_INLINE inline bool freezingChecksEnabled() noexcept { return Kotlin_freezingChecksEnabled != 0; } -ALWAYS_INLINE inline GCSchedulerType getGCSchedulerType() noexcept { - return static_cast(Kotlin_gcSchedulerType); -} - - WorkerExceptionHandling workerExceptionHandling() noexcept; DestroyRuntimeMode destroyRuntimeMode() noexcept; bool gcMarkSingleThreaded() noexcept; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index b6d7b25bd07..ad5a64ca6d6 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -28,6 +28,7 @@ public: GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; } ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } + gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } gc::GC& gc() noexcept { return gc_; } AppStateTracking& appStateTracking() noexcept { return appStateTracking_; } @@ -43,7 +44,8 @@ private: GlobalsRegistry globalsRegistry_; SpecialRefRegistry specialRefRegistry_; ExtraObjectDataFactory extraObjectDataFactory_; - gc::GC gc_; + gcScheduler::GCScheduler gcScheduler_; + gc::GC gc_{gcScheduler_}; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 388c8c8a9f0..584e5f7d589 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -343,11 +343,11 @@ extern "C" void Kotlin_native_internal_GC_start(ObjHeader*) { extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, KInt value) { RuntimeAssert(value > 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = value; + mm::GlobalData::Instance().gcScheduler().config().threshold = value; } extern "C" KInt Kotlin_native_internal_GC_getThreshold(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load(); + return mm::GlobalData::Instance().gcScheduler().config().threshold.load(); } extern "C" void Kotlin_native_internal_GC_setCollectCyclesThreshold(ObjHeader*, int64_t value) { @@ -363,64 +363,64 @@ extern "C" int64_t Kotlin_native_internal_GC_getCollectCyclesThreshold(ObjHeader extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, int64_t value) { RuntimeAssert(value > 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = value; + mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = value; } extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load(); + return mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load(); } extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) { - mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune = value; + mm::GlobalData::Instance().gcScheduler().config().autoTune = value; } extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune.load(); + return mm::GlobalData::Instance().gcScheduler().config().autoTune.load(); } extern "C" KLong Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds.load(); + return mm::GlobalData::Instance().gcScheduler().config().regularGcIntervalMicroseconds.load(); } extern "C" void Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds(ObjHeader*, KLong value) { RuntimeAssert(value >= 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds = value; + mm::GlobalData::Instance().gcScheduler().config().regularGcIntervalMicroseconds = value; } extern "C" KLong Kotlin_native_internal_GC_getTargetHeapBytes(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes.load(); + return mm::GlobalData::Instance().gcScheduler().config().targetHeapBytes.load(); } extern "C" void Kotlin_native_internal_GC_setTargetHeapBytes(ObjHeader*, KLong value) { RuntimeAssert(value >= 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes = value; + mm::GlobalData::Instance().gcScheduler().config().targetHeapBytes = value; } extern "C" KDouble Kotlin_native_internal_GC_getTargetHeapUtilization(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization.load(); + return mm::GlobalData::Instance().gcScheduler().config().targetHeapUtilization.load(); } extern "C" void Kotlin_native_internal_GC_setTargetHeapUtilization(ObjHeader*, KDouble value) { RuntimeAssert(value > 0 && value <= 1, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization = value; + mm::GlobalData::Instance().gcScheduler().config().targetHeapUtilization = value; } extern "C" KLong Kotlin_native_internal_GC_getMaxHeapBytes(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes.load(); + return mm::GlobalData::Instance().gcScheduler().config().maxHeapBytes.load(); } extern "C" void Kotlin_native_internal_GC_setMaxHeapBytes(ObjHeader*, KLong value) { RuntimeAssert(value >= 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes = value; + mm::GlobalData::Instance().gcScheduler().config().maxHeapBytes = value; } extern "C" KLong Kotlin_native_internal_GC_getMinHeapBytes(ObjHeader*) { - return mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes.load(); + return mm::GlobalData::Instance().gcScheduler().config().minHeapBytes.load(); } extern "C" void Kotlin_native_internal_GC_setMinHeapBytes(ObjHeader*, KLong value) { RuntimeAssert(value >= 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes = value; + mm::GlobalData::Instance().gcScheduler().config().minHeapBytes = value; } extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) { @@ -543,12 +543,14 @@ extern "C" void CheckGlobalsAccessible() { extern "C" RUNTIME_NOTHROW NO_INLINE void Kotlin_mm_safePointFunctionPrologue() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); + threadData->gcScheduler().OnSafePointRegular(gcScheduler::GCSchedulerThreadData::kFunctionPrologueWeight); threadData->gc().SafePointFunctionPrologue(); } extern "C" RUNTIME_NOTHROW CODEGEN_INLINE_POLICY void Kotlin_mm_safePointWhileLoopBody() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); + threadData->gcScheduler().OnSafePointRegular(gcScheduler::GCSchedulerThreadData::kLoopBodyWeight); threadData->gc().SafePointLoopBody(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 1ad8101440d..b58af879f51 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -35,7 +35,8 @@ public: globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()), - gc_(GlobalData::Instance().gc(), *this), + gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), + gc_(GlobalData::Instance().gc(), gcScheduler_, *this), suspensionData_(ThreadState::kNative, *this) {} ~ThreadData() = default; @@ -58,6 +59,8 @@ public: std_support::vector>& initializingSingletons() noexcept { return initializingSingletons_; } + gcScheduler::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } + gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -84,6 +87,7 @@ private: SpecialRefRegistry::ThreadQueue specialRefRegistry_; ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_; ShadowStack shadowStack_; + gcScheduler::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_; std_support::vector> initializingSingletons_; ThreadSuspensionData suspensionData_; diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index 9fb718b7c52..470c60a5983 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -74,7 +74,6 @@ extern const int32_t Kotlin_disableMmap = 1; extern const int32_t Kotlin_disableMmap = 0; #endif extern const char* const Kotlin_runtimeLogs = nullptr; -extern const int32_t Kotlin_gcSchedulerType = static_cast(kotlin::compiler::GCSchedulerType::kDisabled); extern const int32_t Kotlin_freezingChecksEnabled = 1; extern const int32_t Kotlin_freezingEnabled = 1;