diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp index 3a1951d3872..02153f40c6c 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp @@ -5,6 +5,8 @@ #include "GCScheduler.hpp" +#include + #include "CompilerConstants.hpp" #include "GlobalData.hpp" #include "KAssert.h" @@ -17,6 +19,72 @@ using namespace kotlin; namespace { +class HeapGrowthController { +public: + explicit HeapGrowthController(gc::GCSchedulerConfig& config) noexcept : config_(config) {} + + // Called by the mutators. + void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; } + + // Called by the GC thread. + void OnPerformFullGC() noexcept { allocatedBytes_ = 0; } + + // Called by the GC thread. + void UpdateAliveSetBytes(size_t bytes) noexcept { + lastAliveSetBytes_ = bytes; + + if (config_.autoTune.load()) { + double targetHeapBytes = static_cast(bytes) / config_.targetHeapUtilization; + if (!std::isfinite(targetHeapBytes)) { + // This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything. + return; + } + double minHeapBytes = static_cast(config_.minHeapBytes.load()); + double maxHeapBytes = static_cast(config_.maxHeapBytes.load()); + targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes); + config_.targetHeapBytes = static_cast(targetHeapBytes); + } + } + + // Called by the mutators. + bool NeedsGC() const noexcept { + uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load(); + uint64_t targetHeapBytes = config_.targetHeapBytes; + return currentHeapBytes >= targetHeapBytes; + } + +private: + gc::GCSchedulerConfig& config_; + // Updated by both the mutators and the GC thread. + std::atomic allocatedBytes_ = 0; + // Updated by the GC thread, read by the mutators. + std::atomic lastAliveSetBytes_ = 0; +}; + +class RegularIntervalPacer { +public: + using TimePoint = std::chrono::time_point; + using CurrentTimeProvider = std::function; + + RegularIntervalPacer(gc::GCSchedulerConfig& config, CurrentTimeProvider currentTimeProvider) noexcept : + config_(config), currentTimeProvider_(currentTimeProvider), lastGC_(currentTimeProvider_()) {} + + // Called by the mutators or the timer thread. + bool NeedsGC() const noexcept { + auto currentTimeProvider = currentTimeProvider_(); + return currentTimeProvider >= lastGC_.load() + config_.regularGcInterval(); + } + + // Called by the GC thread. + void OnPerformFullGC() noexcept { lastGC_ = currentTimeProvider_(); } + +private: + gc::GCSchedulerConfig& config_; + CurrentTimeProvider currentTimeProvider_; + // Updated by the GC thread, read by the mutators or the timer thread. + std::atomic lastGC_; +}; + class GCEmptySchedulerData : public gc::GCSchedulerData { void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {} void OnPerformFullGC() noexcept override {} @@ -25,74 +93,70 @@ class GCEmptySchedulerData : public gc::GCSchedulerData { class GCSchedulerDataWithTimer : public gc::GCSchedulerData { public: - explicit GCSchedulerDataWithTimer(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept : - config_(config), scheduleGC_(std::move(scheduleGC)), timer_(std::chrono::microseconds(config_.regularGcIntervalUs), [this]() { - OnTimer(); - return std::chrono::microseconds(config_.regularGcIntervalUs); + GCSchedulerDataWithTimer( + gc::GCSchedulerConfig& config, + std::function scheduleGC, + RegularIntervalPacer::CurrentTimeProvider currentTimeProvider) noexcept : + config_(config), + heapGrowthController_(config), + regularIntervalPacer_(config, currentTimeProvider), + scheduleGC_(std::move(scheduleGC)), + timer_(config_.regularGcInterval(), [this]() { + if (regularIntervalPacer_.NeedsGC()) { + scheduleGC_(); + } + return config_.regularGcInterval(); }) {} void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { - size_t allocatedBytes = threadData.allocatedBytes(); - if (allocatedBytes > config_.allocationThresholdBytes) { - RuntimeAssert(static_cast(scheduleGC_), "scheduleGC_ cannot be empty"); + heapGrowthController_.OnAllocated(threadData.allocatedBytes()); + if (heapGrowthController_.NeedsGC()) { scheduleGC_(); } } - void OnPerformFullGC() noexcept override {} - - void UpdateAliveSetBytes(size_t bytes) noexcept override {} - -private: - void OnTimer() noexcept { - auto allThreadsAreNative = []() { - auto threadRegistryIter = mm::GlobalData::Instance().threadRegistry().LockForIter(); - return std::all_of(threadRegistryIter.begin(), threadRegistryIter.end(), [](mm::ThreadData& thread) { - return thread.state() == ThreadState::kNative; - }); - }(); - // Don't run, if kotlin code is not being executed. - if (allThreadsAreNative) return; - - // TODO: Probably makes sense to check memory usage of the process. - scheduleGC_(); + void OnPerformFullGC() noexcept override { + heapGrowthController_.OnPerformFullGC(); + regularIntervalPacer_.OnPerformFullGC(); } - gc::GCSchedulerConfig& config_; + void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } +private: + gc::GCSchedulerConfig& config_; + HeapGrowthController heapGrowthController_; + RegularIntervalPacer regularIntervalPacer_; std::function scheduleGC_; RepeatedTimer timer_; }; -class GCSchedulerDataWithoutTimer : public gc::GCSchedulerData { +class GCSchedulerDataOnSafepoints : public gc::GCSchedulerData { public: - using CurrentTimeCallback = std::function; - - GCSchedulerDataWithoutTimer( - gc::GCSchedulerConfig& config, std::function scheduleGC, CurrentTimeCallback currentTimeCallbackNs) noexcept : - config_(config), - currentTimeCallbackNs_(std::move(currentTimeCallbackNs)), - timeOfLastGcNs_(currentTimeCallbackNs_()), - scheduleGC_(std::move(scheduleGC)) {} + GCSchedulerDataOnSafepoints( + gc::GCSchedulerConfig& config, + std::function scheduleGC, + RegularIntervalPacer::CurrentTimeProvider currentTimeProvider) noexcept : + heapGrowthController_(config), regularIntervalPacer_(config, currentTimeProvider), scheduleGC_(std::move(scheduleGC)) {} void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { - size_t allocatedBytes = threadData.allocatedBytes(); - if (allocatedBytes > config_.allocationThresholdBytes || - currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs) { - RuntimeAssert(static_cast(scheduleGC_), "scheduleGC_ cannot be empty"); + heapGrowthController_.OnAllocated(threadData.allocatedBytes()); + if (heapGrowthController_.NeedsGC()) { + scheduleGC_(); + } else if (regularIntervalPacer_.NeedsGC()) { scheduleGC_(); } } - void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); } + void OnPerformFullGC() noexcept override { + heapGrowthController_.OnPerformFullGC(); + regularIntervalPacer_.OnPerformFullGC(); + } - void UpdateAliveSetBytes(size_t bytes) noexcept override {} + void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } private: - gc::GCSchedulerConfig& config_; - CurrentTimeCallback currentTimeCallbackNs_; - - std::atomic timeOfLastGcNs_; + HeapGrowthController heapGrowthController_; + RegularIntervalPacer regularIntervalPacer_; std::function scheduleGC_; }; @@ -115,44 +179,30 @@ private: std::function scheduleGC_; }; -KStdUniquePtr MakeEmptyGCSchedulerData() noexcept { - return ::make_unique(); -} - -KStdUniquePtr MakeGCSchedulerDataWithTimer( - gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept { - return ::make_unique(config, std::move(scheduleGC)); -} - -KStdUniquePtr MakeGCSchedulerDataWithoutTimer( - gc::GCSchedulerConfig& config, std::function scheduleGC, std::function currentTimeCallbackNs) noexcept { - return ::make_unique(config, std::move(scheduleGC), std::move(currentTimeCallbackNs)); -} - -KStdUniquePtr MakeGCShedulerDataAggressive(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept { - return ::make_unique(config, std::move(scheduleGC)); -} - } // namespace -KStdUniquePtr kotlin::gc::MakeGCSchedulerData(SchedulerType type, gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept { +KStdUniquePtr kotlin::gc::internal::MakeGCSchedulerData( + SchedulerType type, + gc::GCSchedulerConfig& config, + std::function scheduleGC, + std::function()> currentTimeProvider) noexcept { switch (type) { case SchedulerType::kDisabled: - return MakeEmptyGCSchedulerData(); + return ::make_unique(); case SchedulerType::kWithTimer: - return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC)); + return ::make_unique(config, std::move(scheduleGC), std::move(currentTimeProvider)); case SchedulerType::kOnSafepoints: - return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); }); + return ::make_unique(config, std::move(scheduleGC), std::move(currentTimeProvider)); case SchedulerType::kAggressive: - return MakeGCShedulerDataAggressive(config, std::move(scheduleGC)); + return ::make_unique(config, std::move(scheduleGC)); } } - void gc::GCScheduler::SetScheduleGC(std::function scheduleGC) noexcept { RuntimeAssert(static_cast(scheduleGC), "scheduleGC cannot be empty"); RuntimeAssert(!static_cast(scheduleGC_), "scheduleGC must not have been set"); scheduleGC_ = std::move(scheduleGC); RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call"); - gcData_ = MakeGCSchedulerData(compiler::getGCSchedulerType(), config_, scheduleGC_); + gcData_ = internal::MakeGCSchedulerData( + compiler::getGCSchedulerType(), config_, scheduleGC_, []() { return std::chrono::steady_clock::now(); }); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp index 86c8bdf57d8..eb82a7fbb6d 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp @@ -7,6 +7,7 @@ #define RUNTIME_GC_COMMON_GC_SCHEDULER_H #include +#include #include #include #include @@ -22,13 +23,28 @@ namespace gc { using SchedulerType = compiler::GCSchedulerType; - +// NOTE: When changing default values, reflect them in GC.kt as well. struct GCSchedulerConfig { - std::atomic threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine). - std::atomic allocationThresholdBytes = 10 * 1024 * 1024; // 10MiB by default. - std::atomic cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default. - std::atomic autoTune = false; - std::atomic regularGcIntervalUs = 200 * 1000; // 200 milliseconds by default. + // Only used when useGCTimer() is false. How many regular safepoints will trigger slow path. + std::atomic threshold = 100000; + // How many object bytes a thread must allocate to trigger slow path. + std::atomic allocationThresholdBytes = 10 * 1024; + std::atomic autoTune = true; + // The target interval between collections when Kotlin code is idle. GC will be triggered + // by timer no sooner than this value and no later than twice this value since the previous collection. + std::atomic regularGcIntervalMicroseconds = 10 * 1000 * 1000; + // How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true. + std::atomic targetHeapBytes = 1024 * 1024; + // The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection + // N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped + // between minHeapBytes and maxHeapBytes. + std::atomic targetHeapUtilization = 0.5; + // The minimum value of targetHeapBytes for autoTune = true + std::atomic minHeapBytes = 1024 * 1024; + // The maximum value of targetHeapBytes for autoTune = true + std::atomic maxHeapBytes = std::numeric_limits::max(); + + std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); } }; class GCSchedulerThreadData; @@ -40,6 +56,10 @@ public: // Called by different mutator threads. virtual void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept = 0; + // The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC` + // when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished. + // TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead. + // Always called by the GC thread. virtual void OnPerformFullGC() noexcept = 0; @@ -63,11 +83,7 @@ public: switch (compiler::getGCSchedulerType()) { case compiler::GCSchedulerType::kOnSafepoints: case compiler::GCSchedulerType::kAggressive: - safePointsCounter_ += weight; - if (safePointsCounter_ < safePointsCounterThreshold_) { - return; - } - OnSafePointSlowPath(); + OnSafePointRegularImpl(weight); return; default: return; @@ -91,6 +107,16 @@ public: size_t safePointsCounter() const noexcept { return safePointsCounter_; } private: + friend class GCSchedulerThreadDataTestApi; + + void OnSafePointRegularImpl(size_t weight) noexcept { + safePointsCounter_ += weight; + if (safePointsCounter_ < safePointsCounterThreshold_) { + return; + } + OnSafePointSlowPath(); + } + void OnSafePointSlowPath() noexcept { slowPath_(*this); ClearCountersAndUpdateThresholds(); @@ -113,7 +139,6 @@ private: size_t safePointsCounterThreshold_ = 0; }; - class GCScheduler : private Pinned { public: GCScheduler() noexcept = default; @@ -138,10 +163,15 @@ private: std::function scheduleGC_; }; +namespace internal { + KStdUniquePtr MakeGCSchedulerData( SchedulerType type, GCSchedulerConfig& config, - std::function scheduleGC) noexcept; + std::function scheduleGC, + std::function()> currentTime) noexcept; + +} } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp new file mode 100644 index 00000000000..df4931038b2 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GCSchedulerTest.cpp @@ -0,0 +1,725 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCScheduler.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "SingleThreadExecutor.hpp" +#include "TestSupport.hpp" + +using namespace kotlin; + +using testing::_; + +namespace kotlin { +namespace gc { + +class GCSchedulerThreadDataTestApi : private Pinned { +public: + explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {} + + void OnSafePointRegularImpl(size_t weight) { scheduler_.OnSafePointRegularImpl(weight); } + + void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; } + +private: + GCSchedulerThreadData& scheduler_; +}; + +TEST(GCSchedulerThreadDataTest, RegularSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = 1; + config.threshold = kThreshold; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); + }); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), kWeight); +} + +TEST(GCSchedulerThreadDataTest, AllocationSafePoint) { + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = 1; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + }); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kSize); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, ResetByGC) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + + EXPECT_CALL(slowPath, Call(_)).Times(0); + scheduler.OnStoppedForGC(); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + scheduler.OnStoppedForGC(); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterRegularSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold); + }); + schedulerTestApi.OnSafePointRegularImpl(kWeight); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) { + constexpr size_t kWeight = 2; + constexpr size_t kSize = 2; + constexpr size_t kCount = 10; + constexpr size_t kThreshold = kCount * kWeight; + constexpr size_t kAllocationThreshold = kCount * kSize; + testing::MockFunction slowPath; + GCSchedulerConfig config; + config.allocationThresholdBytes = kAllocationThreshold; + config.threshold = kThreshold; + GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); + GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); + + config.allocationThresholdBytes = kAllocationThreshold - kSize; + config.threshold = kThreshold - kWeight; + + EXPECT_CALL(slowPath, Call(_)).Times(0); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); + }); + scheduler.OnSafePointAllocation(kSize); + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + schedulerTestApi.OnSafePointRegularImpl(kWeight); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); + + EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) { + EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); + }); + for (size_t i = 0; i < kCount - 1; ++i) { + scheduler.OnSafePointAllocation(kSize); + } + testing::Mock::VerifyAndClearExpectations(&slowPath); + EXPECT_THAT(scheduler.allocatedBytes(), 0); + EXPECT_THAT(scheduler.safePointsCounter(), 0); +} + +using TimePoint = std::chrono::time_point; + +class MutatorThread : private Pinned { +public: + MutatorThread(GCSchedulerConfig& config, std::function slowPath) : + executor_(MakeSingleThreadExecutorWithContext( + [&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); })) {} + + std::future Allocate(size_t bytes) { + return executor_.Execute([&, bytes] { + auto& context = executor_.thread().context(); + context.threadDataTestApi.SetAllocatedBytes(bytes); + context.slowPath(context.threadData); + }); + } + +private: + struct Context { + GCSchedulerThreadData threadData; + GCSchedulerThreadDataTestApi threadDataTestApi; + std::function slowPath; + + Context(GCSchedulerConfig& config, std::function slowPath) : + threadData(config, [](GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {} + }; + + SingleThreadExecutor> executor_; +}; + +template +class GCSchedulerDataTestApi { +public: + static constexpr TimePoint initialTime = TimePoint(); + + explicit GCSchedulerDataTestApi(GCSchedulerConfig& config) { + ON_CALL(currentTime_, Call()).WillByDefault([&]() { return time_.load(); }); + + scheduler_ = internal::MakeGCSchedulerData(schedulerType, config, scheduleGC_.AsStdFunction(), currentTime_.AsStdFunction()); + + mutators_.reserve(MutatorCount); + for (int i = 0; i < MutatorCount; ++i) { + mutators_.emplace_back(make_unique( + config, [this](GCSchedulerThreadData& threadData) { scheduler_->UpdateFromThreadData(threadData); })); + } + } + + std::future Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); } + + void OnPerformFullGC() { scheduler_->OnPerformFullGC(); } + + void UpdateAliveSetBytes(size_t bytes) { scheduler_->UpdateAliveSetBytes(bytes); } + + testing::MockFunction& scheduleGC() { return scheduleGC_; } + + template + void advance_time(Duration duration) { + auto time = time_.load(); + while (true) { + auto newTime = time + std::chrono::duration_cast(duration); + if (time_.compare_exchange_weak(time, newTime)) { + // TODO: Figure out mocking out RepeatedTimer (or clock underneath it) to avoid sleeping. + std::this_thread::sleep_for(duration); + return; + } + } + } + +private: + std::atomic time_ = initialTime; + KStdVector> mutators_; + testing::MockFunction scheduleGC_; + testing::NiceMock> currentTime_; + KStdUniquePtr scheduler_; +}; + +TEST(GCSchedulerDataOnSafePoints, CollectOnTargetHeapReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = false; + config.targetHeapBytes = (mutatorsCount + 1) * 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + KStdVector> futures; + for (int i = 0; i < mutatorsCount; ++i) { + futures.push_back(schedulerTestApi.Allocate(i, 10)); + } + for (auto& future : futures) { + future.get(); + } + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataOnSafePoints, CollectOnTimeoutReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(20)).count(); + config.autoTune = false; + config.targetHeapBytes = std::numeric_limits::max(); + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(10)); + KStdVector> futures; + for (int i = 0; i < mutatorsCount; ++i) { + futures.push_back(schedulerTestApi.Allocate(i, 0)); + } + for (auto& future : futures) { + future.get(); + } + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(std::chrono::milliseconds(15)); + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataOnSafePoints, FullTimeoutAfterLastGC) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(20)).count(); + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(std::chrono::milliseconds(10)); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(15)); + schedulerTestApi.Allocate(0, 0).get(); + // It's now 25 ms since the start, but only 15ms since previous collection. + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(std::chrono::milliseconds(10)); + schedulerTestApi.Allocate(0, 0).get(); + // It's now 25 ms since the previous collection. + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataOnSafePoints, DoNotTuneTargetHeap) { + constexpr int mutatorsCount = 1; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 10); +} + +TEST(GCSchedulerDataOnSafePoints, TuneTargetHeap) { + constexpr int mutatorsCount = 1; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = true; + config.targetHeapBytes = 10; + config.targetHeapUtilization = 0.5; + config.minHeapBytes = 5; + config.maxHeapBytes = 50; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 20); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 20. + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(20); + + EXPECT_THAT(config.targetHeapBytes.load(), 40); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 60. + schedulerTestApi.Allocate(0, 40).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // Keeping total heap of 60. + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 40 + schedulerTestApi.UpdateAliveSetBytes(40); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 50 + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 1 + schedulerTestApi.UpdateAliveSetBytes(1); + + // But the minimum is set to 5. + EXPECT_THAT(config.targetHeapBytes.load(), 5); +} + +TEST(GCSchedulerDataWithTimer, CollectOnTargetHeapReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = false; + config.targetHeapBytes = (mutatorsCount + 1) * 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + KStdVector> futures; + for (int i = 0; i < mutatorsCount; ++i) { + futures.push_back(schedulerTestApi.Allocate(i, 10)); + } + for (auto& future : futures) { + future.get(); + } + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataWithTimer, CollectOnTimeoutReached) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(200)).count(); + config.autoTune = false; + config.targetHeapBytes = std::numeric_limits::max(); + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(100)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(std::chrono::milliseconds(150)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataWithTimer, FullTimeoutAfterLastGC) { + constexpr int mutatorsCount = kDefaultThreadCount; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(200)).count(); + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(100)); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(150)); + // It's now 250 ms since the start, but only 150ms since previous collection. + // However, the timer has fired once ~50ms ago. + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.advance_time(std::chrono::milliseconds(100)); + // It's now 250 ms since the previous collection, but the timer will fire in ~50ms. + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.advance_time(std::chrono::milliseconds(100)); + // 350ms since previous collection, and the timer has fired ~50ms ago. + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(0); +} + +TEST(GCSchedulerDataWithTimer, DoNotTuneTargetHeap) { + constexpr int mutatorsCount = 1; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = false; + config.targetHeapBytes = 10; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 10); +} + +TEST(GCSchedulerDataWithTimer, TuneTargetHeap) { + constexpr int mutatorsCount = 1; + + GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count(); + config.autoTune = true; + config.targetHeapBytes = 10; + config.targetHeapUtilization = 0.5; + config.minHeapBytes = 5; + config.maxHeapBytes = 50; + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(10); + + EXPECT_THAT(config.targetHeapBytes.load(), 20); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 20. + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(20); + + EXPECT_THAT(config.targetHeapBytes.load(), 40); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 60. + schedulerTestApi.Allocate(0, 40).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // Keeping total heap of 60. + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.UpdateAliveSetBytes(60); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + schedulerTestApi.Allocate(0, 0).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 40 + schedulerTestApi.UpdateAliveSetBytes(40); + + EXPECT_THAT(config.targetHeapBytes.load(), 50); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + // For a total heap of 50 + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + schedulerTestApi.OnPerformFullGC(); + // Dropping to 1 + schedulerTestApi.UpdateAliveSetBytes(1); + + // But the minimum is set to 5. + EXPECT_THAT(config.targetHeapBytes.load(), 5); +} + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index ba59bd40c75..b458f078c4b 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -105,8 +105,12 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), gcScheduler_(gcScheduler) { gcScheduler_.SetScheduleGC([]() { - RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); - gSafepointFlag = SafepointFlag::kNeedsGC; + // TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running. + // Investigate, if it's possible to move this logic into the scheduler. + SafepointFlag expectedFlag = SafepointFlag::kNone; + if (gSafepointFlag.compare_exchange_strong(expectedFlag, SafepointFlag::kNeedsGC)) { + RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); + } }); } diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index cfb235c9a17..f05dac9cb6c 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -2635,10 +2635,8 @@ void startGC() { } void setGCThreshold(KInt value) { + RuntimeAssert(value > 0, "Must be handled by the caller"); GC_LOG("setGCThreshold %d\n", value) - if (value <= 0) { - ThrowIllegalArgumentException(); - } initGcThreshold(memoryState, value); } @@ -2648,10 +2646,8 @@ KInt getGCThreshold() { } void setGCCollectCyclesThreshold(KLong value) { + RuntimeAssert(value > 0, "Must be handled by the caller"); GC_LOG("setGCCollectCyclesThreshold %lld\n", value) - if (value <= 0) { - ThrowIllegalArgumentException(); - } initGcCollectCyclesThreshold(memoryState, value); } @@ -2661,10 +2657,8 @@ KInt getGCCollectCyclesThreshold() { } void setGCThresholdAllocations(KLong value) { + RuntimeAssert(value > 0, "Must be handled by the caller"); GC_LOG("setGCThresholdAllocations %lld\n", value) - if (value <= 0) { - ThrowIllegalArgumentException(); - } memoryState->allocSinceLastGcThreshold = value; } @@ -3615,6 +3609,36 @@ OBJ_GETTER(Kotlin_native_internal_GC_findCycle, KRef, KRef root) { #endif } +KLong Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds(KRef) { + return 0; +} + +void Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds(KRef, KLong) {} + +KLong Kotlin_native_internal_GC_getTargetHeapBytes(KRef) { + return 0; +} + +void Kotlin_native_internal_GC_setTargetHeapBytes(KRef, KLong) {} + +KDouble Kotlin_native_internal_GC_getTargetHeapUtilization(KRef) { + return 1.0; +} + +void Kotlin_native_internal_GC_setTargetHeapUtilization(KRef, KDouble) {} + +KLong Kotlin_native_internal_GC_getMaxHeapBytes(KRef) { + return -1; +} + +void Kotlin_native_internal_GC_setMaxHeapBytes(KRef, KLong) {} + +KLong Kotlin_native_internal_GC_getMinHeapBytes(KRef) { + return 0; +} + +void Kotlin_native_internal_GC_setMinHeapBytes(KRef, KLong) {} + RUNTIME_NOTHROW KNativePtr CreateStablePointer(KRef any) { return createStablePointer(any); } diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/GC.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/GC.kt index 00920609b93..947e7c12318 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/GC.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/GC.kt @@ -5,6 +5,9 @@ package kotlin.native.internal +import kotlin.time.Duration +import kotlin.time.Duration.Companion.microseconds + /** * ## Cycle garbage collector interface. * @@ -25,12 +28,14 @@ object GC { /** * To force garbage collection immediately, unless collector is stopped * with [stop] operation. Even if GC is suspended, [collect] still triggers collection. + * New MM: trigger new collection and wait for its completion. */ @GCUnsafeCall("Kotlin_native_internal_GC_collect") external fun collect() /** * Request global cyclic collector, operation is async and just triggers the collection. + * New MM: unused. */ @GCUnsafeCall("Kotlin_native_internal_GC_collectCyclic") external fun collectCyclic() @@ -38,18 +43,21 @@ object GC { /** * Suspend garbage collection. Release candidates are still collected, but * GC algorithm is not executed. + * New MM: unused. */ @GCUnsafeCall("Kotlin_native_internal_GC_suspend") external fun suspend() /** * Resume garbage collection. Can potentially lead to GC immediately. + * New MM: unused. */ @GCUnsafeCall("Kotlin_native_internal_GC_resume") external fun resume() /** * Stop garbage collection. Cyclical garbage is no longer collected. + * New MM: unused. */ @GCUnsafeCall("Kotlin_native_internal_GC_stop") external fun stop() @@ -57,6 +65,7 @@ object GC { /** * Start garbage collection. Cyclical garbage produced while GC was stopped * cannot be reclaimed, but all new garbage is collected. + * New MM: unused. */ @GCUnsafeCall("Kotlin_native_internal_GC_start") external fun start() @@ -64,29 +73,59 @@ object GC { /** * GC threshold, controlling how frequenly GC is activated, and how much time GC * takes. Bigger values lead to longer GC pauses, but less GCs. + * New MM: usually unused. For the on-safepoints GC scheduler counts + * how many safepoints must the code pass before informing the GC scheduler. + * + * Default: (old MM) 8 * 1024 + * Default: (new MM) 100000 + * + * @throws [IllegalArgumentException] when value is not positive. */ var threshold: Int get() = getThreshold() - set(value) = setThreshold(value) + set(value) { + require(value > 0) { "threshold must be positive: $value" } + setThreshold(value) + } /** * GC allocation threshold, controlling how frequenly GC collect cycles, and how much time * this process takes. Bigger values lead to longer GC pauses, but less GCs. + * New MM: unused. + * + * Default: 8 * 1024 + * + * @throws [IllegalArgumentException] when value is not positive. */ var collectCyclesThreshold: Long get() = getCollectCyclesThreshold() - set(value) = setCollectCyclesThreshold(value) + set(value) { + require(value > 0) { "collectCyclesThreshold must be positive: $value" } + setCollectCyclesThreshold(value) + } /** * GC allocation threshold, controlling how many bytes allocated since last * collection will trigger new GC. + * New MM: how many bytes a thread can allocate before informing the GC scheduler. + * + * Default: (old MM) 8 * 1024 * 1024 + * Default: (new MM) 10 * 1024 + * + * @throws [IllegalArgumentException] when value is not positive. */ var thresholdAllocations: Long get() = getThresholdAllocations() - set(value) = setThresholdAllocations(value) + set(value) { + require(value > 0) { "thresholdAllocations must be positive: $value" } + setThresholdAllocations(value) + } /** * If GC shall auto-tune thresholds, depending on how much time is spent in collection. + * New MM: if true update targetHeapBytes after each collection. + * + * Default: true */ var autotune: Boolean get() = getTuneThreshold() @@ -95,21 +134,110 @@ object GC { /** * If cyclic collector for atomic references to be deployed. + * New MM: unused. */ var cyclicCollectorEnabled: Boolean get() = getCyclicCollectorEnabled() set(value) = setCyclicCollectorEnabled(value) + /** + * New MM only. Unused with on-safepoints GC scheduler. + * When Kotlin code is not allocating enough to trigger GC, the GC scheduler uses timer to drive collection. + * Timer-triggered collection will happen roughly in [regularGCInterval] .. 2 * [regularGCInterval] since + * any previous collection. + * + * Default: 10 seconds + * + * @throws [IllegalArgumentException] when value is negative. + */ + var regularGCInterval: Duration + get() = getRegularGCIntervalMicroseconds().microseconds + set(value) { + require(!value.isNegative()) { "regularGCInterval must not be negative: $value" } + setRegularGCIntervalMicroseconds(value.inWholeMicroseconds) + } + + /** + * New MM only. + * Total amount of heap available for Kotlin objects. When Kotlin objects overflow this heap, + * the garbage collection is requested. Automatically adjusts when [autotune] is true: + * after each collection the [targetHeapBytes] is set to heapBytes / [targetHeapUtilization] and + * capped between [minHeapBytes] and [maxHeapBytes], where heapBytes is heap usage after the garbage + * is collected. + * Note, that if after a collection heapBytes > [targetHeapBytes] (which may happen if [autotune] is false, + * or [maxHeapBytes] is set too low), the next collection will be triggered almost immediately. + * + * Default: 1 MiB + * + * @throws [IllegalArgumentException] when value is negative. + */ + var targetHeapBytes: Long + get() = getTargetHeapBytes() + set(value) { + require(value >= 0) { "targetHeapBytes must not be negative: $value" } + setTargetHeapBytes(value) + } + + /** + * New MM only. + * What fraction of the Kotlin heap should be populated. + * Only used if [autotune] is true. See [targetHeapBytes] for more details. + * + * Default: 0.5 + * + * @throws [IllegalArgumentException] when value is outside (0, 1] interval. + */ + var targetHeapUtilization: Double + get() = getTargetHeapUtilization() + set(value) { + require(value > 0 && value <= 1) { "targetHeapUtilization must be in (0, 1] interval: $value" } + setTargetHeapUtilization(value) + } + + /** + * New MM only. + * The minimum value for [targetHeapBytes] + * Only used if [autotune] is true. See [targetHeapBytes] for more details. + * + * Default: 1 MiB + * + * @throws [IllegalArgumentException] when value is negative. + */ + var minHeapBytes: Long + get() = getMinHeapBytes() + set(value) { + require(value >= 0) { "minHeapBytes must not be negative: $value" } + setMinHeapBytes(value) + } + + /** + * New MM only. + * The maximum value for [targetHeapBytes]. + * Only used if [autotune] is true. See [targetHeapBytes] for more details. + * + * Default: [Long.MAX_VALUE] + * + * @throws [IllegalArgumentException] when value is negative. + */ + var maxHeapBytes: Long + get() = getMaxHeapBytes() + set(value) { + require(value >= 0) { "maxHeapBytes must not be negative: $value" } + setMaxHeapBytes(value) + } + /** * Detect cyclic references going via atomic references and return list of cycle-inducing objects * or `null` if the leak detector is not available. Use [Platform.isMemoryLeakCheckerActive] to check * leak detector availability. + * New MM: unused. Always returns null. */ @GCUnsafeCall("Kotlin_native_internal_GC_detectCycles") external fun detectCycles(): Array? /** * Find a reference cycle including from the given object, `null` if no cycles detected. + * New MM: unused. Always returns null. */ @GCUnsafeCall("Kotlin_native_internal_GC_findCycle") external fun findCycle(root: Any): Array? @@ -143,4 +271,34 @@ object GC { @GCUnsafeCall("Kotlin_native_internal_GC_setCyclicCollector") private external fun setCyclicCollectorEnabled(value: Boolean) + + @GCUnsafeCall("Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds") + private external fun getRegularGCIntervalMicroseconds(): Long + + @GCUnsafeCall("Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds") + private external fun setRegularGCIntervalMicroseconds(value: Long) + + @GCUnsafeCall("Kotlin_native_internal_GC_getTargetHeapBytes") + private external fun getTargetHeapBytes(): Long + + @GCUnsafeCall("Kotlin_native_internal_GC_setTargetHeapBytes") + private external fun setTargetHeapBytes(value: Long) + + @GCUnsafeCall("Kotlin_native_internal_GC_getTargetHeapUtilization") + private external fun getTargetHeapUtilization(): Double + + @GCUnsafeCall("Kotlin_native_internal_GC_setTargetHeapUtilization") + private external fun setTargetHeapUtilization(value: Double) + + @GCUnsafeCall("Kotlin_native_internal_GC_getMinHeapBytes") + private external fun getMinHeapBytes(): Long + + @GCUnsafeCall("Kotlin_native_internal_GC_setMinHeapBytes") + private external fun setMinHeapBytes(value: Long) + + @GCUnsafeCall("Kotlin_native_internal_GC_getMaxHeapBytes") + private external fun getMaxHeapBytes(): Long + + @GCUnsafeCall("Kotlin_native_internal_GC_setMaxHeapBytes") + private external fun setMaxHeapBytes(value: Long) } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 62219090eef..ec663285d79 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -299,7 +299,7 @@ extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) { extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) { // TODO: Remove when legacy MM is gone. - ThrowIllegalArgumentException(); + // Nothing to do } // TODO: Maybe a pair of suspend/resume or start/stop may be useful in the future? @@ -321,46 +321,33 @@ extern "C" void Kotlin_native_internal_GC_start(ObjHeader*) { // Nothing to do } -extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, int32_t value) { - if (value < 0) { - ThrowIllegalArgumentException(); - } - mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = static_cast(value); +extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, KInt value) { + RuntimeAssert(value > 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = value; } -extern "C" int32_t Kotlin_native_internal_GC_getThreshold(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load(); - auto maxValue = std::numeric_limits::max(); - if (threshold > static_cast(maxValue)) { - return maxValue; - } - return static_cast(maxValue); +extern "C" KInt Kotlin_native_internal_GC_getThreshold(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load(); } extern "C" void Kotlin_native_internal_GC_setCollectCyclesThreshold(ObjHeader*, int64_t value) { // TODO: Remove when legacy MM is gone. - ThrowIllegalArgumentException(); + // Nothing to do } extern "C" int64_t Kotlin_native_internal_GC_getCollectCyclesThreshold(ObjHeader*) { // TODO: Remove when legacy MM is gone. - ThrowIllegalArgumentException(); + // Nothing to do + return -1; } extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, int64_t value) { - if (value < 0) { - ThrowIllegalArgumentException(); - } - mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = static_cast(value); + RuntimeAssert(value > 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = value; } extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load(); - auto maxValue = std::numeric_limits::max(); - if (threshold > static_cast(maxValue)) { - return maxValue; - } - return static_cast(maxValue); + return mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load(); } extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) { @@ -371,6 +358,51 @@ extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) { return mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune.load(); } +extern "C" KLong Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds.load(); +} + +extern "C" void Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds(ObjHeader*, KLong value) { + RuntimeAssert(value >= 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds = value; +} + +extern "C" KLong Kotlin_native_internal_GC_getTargetHeapBytes(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes.load(); +} + +extern "C" void Kotlin_native_internal_GC_setTargetHeapBytes(ObjHeader*, KLong value) { + RuntimeAssert(value >= 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes = value; +} + +extern "C" KDouble Kotlin_native_internal_GC_getTargetHeapUtilization(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization.load(); +} + +extern "C" void Kotlin_native_internal_GC_setTargetHeapUtilization(ObjHeader*, KDouble value) { + RuntimeAssert(value > 0 && value <= 1, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization = value; +} + +extern "C" KLong Kotlin_native_internal_GC_getMaxHeapBytes(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes.load(); +} + +extern "C" void Kotlin_native_internal_GC_setMaxHeapBytes(ObjHeader*, KLong value) { + RuntimeAssert(value >= 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes = value; +} + +extern "C" KLong Kotlin_native_internal_GC_getMinHeapBytes(ObjHeader*) { + return mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes.load(); +} + +extern "C" void Kotlin_native_internal_GC_setMinHeapBytes(ObjHeader*, KLong value) { + RuntimeAssert(value >= 0, "Must be handled by the caller"); + mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes = value; +} + extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) { // TODO: Remove when legacy MM is gone. RETURN_OBJ(nullptr); @@ -383,13 +415,13 @@ extern "C" OBJ_GETTER(Kotlin_native_internal_GC_findCycle, ObjHeader*, ObjHeader extern "C" bool Kotlin_native_internal_GC_getCyclicCollector(ObjHeader* gc) { // TODO: Remove when legacy MM is gone. + // Nothing to do. return false; } extern "C" void Kotlin_native_internal_GC_setCyclicCollector(ObjHeader* gc, bool value) { // TODO: Remove when legacy MM is gone. - if (value) - ThrowIllegalArgumentException(); + // Nothing to do. } extern "C" bool Kotlin_Any_isShareable(ObjHeader* thiz) {