diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp index 6d06f1efa51..bdc49a32af9 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp @@ -5,7 +5,6 @@ #include "GCSchedulerImpl.hpp" -#include "CallsChecker.hpp" #include "GlobalData.hpp" #include "Memory.h" #include "Logging.hpp" @@ -23,9 +22,6 @@ gcScheduler::GCScheduler::ThreadData::~ThreadData() = default; gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept : impl_(config, []() noexcept { - // This call acquires a lock, but the lock are always short-lived, - // so we ignore thread state switching to avoid recursive safe points. - CallsCheckerIgnoreGuard guard; return mm::GlobalData::Instance().gc().Schedule(); }) {} diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp index a80abf91660..0d8940e45de 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -16,6 +16,7 @@ #include "RegularIntervalPacer.hpp" #include "RepeatedTimer.hpp" #include "SafePoint.hpp" +#include "EpochScheduler.hpp" #include "ThreadData.hpp" namespace kotlin::gcScheduler { @@ -55,7 +56,7 @@ public: } if (regularIntervalPacer_.NeedsGC()) { RuntimeLogDebug({kTagGC}, "Scheduling GC by timer"); - schedule(); + scheduleGC_.scheduleNextEpochIfNotInProgress(); } }) { RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized"); @@ -73,18 +74,19 @@ public: return; case HeapGrowthController::MemoryBoundary::kTrigger: RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - schedule(); + scheduleGC_.scheduleNextEpochIfNotInProgress(); return; case HeapGrowthController::MemoryBoundary::kTarget: RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - auto epoch = schedule(); - RuntimeLogWarning({kTagGC}, "Pausing the mutators"); + auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress(); + RuntimeLogWarning({kTagGC}, "Pausing the mutators until epoch %" PRId64 " is done", epoch); mutatorAssists_.requestAssists(epoch); return; } } void onGCFinish(int64_t epoch, size_t bytes) noexcept { + scheduleGC_.onGCFinish(epoch); heapGrowthController_.updateBoundaries(bytes); // Must wait for all mutators to be released. GC thread cannot continue. // This is the contract between GC and mutators. With regular native state @@ -95,13 +97,13 @@ public: }); } - int64_t schedule() noexcept { return scheduleGC_(); } + int64_t schedule() noexcept { return scheduleGC_.scheduleNextEpoch(); } MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; } private: GCSchedulerConfig& config_; - std::function scheduleGC_; + EpochScheduler scheduleGC_; mm::AppStateTracking& appStateTracking_; HeapGrowthController heapGrowthController_; RegularIntervalPacer regularIntervalPacer_; diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp index 5368d350f42..68d16b0546f 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp @@ -176,7 +176,7 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) { // Wait until the timer is initialized. test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); schedulerTestApi.advance_time(microseconds(10)); test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); @@ -197,7 +197,7 @@ TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) { test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); schedulerTestApi.advance_time(microseconds(5)); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); @@ -218,7 +218,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotTuneTargetHeap) { config.targetHeapBytes = 10; GCSchedulerDataTestApi schedulerTestApi(config); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); @@ -239,7 +239,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { config.maxHeapBytes = 50; GCSchedulerDataTestApi schedulerTestApi(config); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); @@ -247,7 +247,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { EXPECT_THAT(config.targetHeapBytes.load(), 20); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2)); // For a total heap of 20. schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); @@ -256,7 +256,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { EXPECT_THAT(config.targetHeapBytes.load(), 40); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(3)); // For a total heap of 60. schedulerTestApi.Allocate(0, 40).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); @@ -266,7 +266,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down EXPECT_THAT(config.targetHeapBytes.load(), 50); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(4)); // Keeping total heap of 60. schedulerTestApi.Allocate(0, 0).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); @@ -275,7 +275,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { EXPECT_THAT(config.targetHeapBytes.load(), 50); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(5)); schedulerTestApi.Allocate(0, 0).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); @@ -284,7 +284,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { EXPECT_THAT(config.targetHeapBytes.load(), 50); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(6)); // For a total heap of 50 schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); @@ -324,7 +324,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotCollectOnTimerInBackground) { // Now go back into the foreground. appStateTracking.setState(mm::AppStateTracking::State::kForeground); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); schedulerTestApi.advance_time(microseconds(10)); test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp index 038a15814d1..c3b21c09f66 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp @@ -5,7 +5,6 @@ #include "GCSchedulerImpl.hpp" -#include "CallsChecker.hpp" #include "GlobalData.hpp" #include "Memory.h" #include "Logging.hpp" @@ -23,9 +22,6 @@ gcScheduler::GCScheduler::ThreadData::~ThreadData() = default; gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept : impl_(config, []() noexcept { - // This call acquires a lock, but the lock are always short-lived, - // so we ignore thread state switching to avoid recursive safe points. - CallsCheckerIgnoreGuard guard; return mm::GlobalData::Instance().gc().Schedule(); }) {} diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp index 02295682c5f..6ef5cd53feb 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp @@ -15,6 +15,7 @@ #include "MutatorAssists.hpp" #include "SafePoint.hpp" #include "SafePointTracker.hpp" +#include "EpochScheduler.hpp" #include "ThreadData.hpp" namespace kotlin::gcScheduler { @@ -56,11 +57,11 @@ public: return; case HeapGrowthController::MemoryBoundary::kTrigger: RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - schedule(); + scheduleGC_.scheduleNextEpochIfNotInProgress(); return; case HeapGrowthController::MemoryBoundary::kTarget: RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - auto epoch = schedule(); + auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress(); RuntimeLogWarning({kTagGC}, "Pausing the mutators"); mutatorAssists_.requestAssists(epoch); return; @@ -75,6 +76,7 @@ public: } void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept { + scheduleGC_.onGCFinish(epoch); heapGrowthController_.updateBoundaries(aliveBytes); // Must wait for all mutators to be released. GC thread cannot continue. // This is the contract between GC and mutators. With regular native state @@ -85,12 +87,12 @@ public: }); } - int64_t schedule() noexcept { return scheduleGC_(); } + int64_t schedule() noexcept { return scheduleGC_.scheduleNextEpoch(); } MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; } private: - std::function scheduleGC_; + EpochScheduler scheduleGC_; HeapGrowthController heapGrowthController_; SafePointTracker<> safePointTracker_; mm::SafePointActivator safePointActivator_; diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp index a507c382639..f86b4347499 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp @@ -56,25 +56,36 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); int i = 0; + std::optional scheduled; // We trigger GC on the first iteration, when the unique allocation point is faced, // on the second to last iteration when weak target heap size is reached, // and on the last iteration when target heap size is reached. EXPECT_CALL(scheduleGC, Call()) - .WillOnce([&i]() { + .WillOnce([&]() { EXPECT_THAT(i, 0); - return 0; - }) - .WillOnce([&i]() { - EXPECT_THAT(i, 8); + EXPECT_THAT(scheduled, std::nullopt); + scheduled = 1; return 1; }) - .WillOnce([&i]() { - EXPECT_THAT(i, 9); + .WillOnce([&]() { + EXPECT_THAT(i, 8); + EXPECT_THAT(scheduled, std::nullopt); + scheduled = 2; return 2; + }) + .WillOnce([&]() { + EXPECT_THAT(i, 9); + EXPECT_THAT(scheduled, std::nullopt); + scheduled = 3; + return 3; }); for (; i < 10; i++) { scheduler.setAllocatedBytes(i + 1); + if (scheduled) { + scheduler.onGCFinish(*scheduled, i + 1); + scheduled = std::nullopt; + } } testing::Mock::VerifyAndClearExpectations(&scheduleGC); }(); diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.cpp new file mode 100644 index 00000000000..cc134a778da --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "EpochScheduler.hpp" + +#include + +#include "CallsChecker.hpp" +#include "KAssert.h" + +using namespace kotlin; + +using Epoch = gcScheduler::internal::EpochScheduler::Epoch; + +Epoch gcScheduler::internal::EpochScheduler::scheduleNextEpoch() noexcept { + // The locks here are always short-lived, + // so we ignore thread state switching to avoid recursive safe points. + CallsCheckerIgnoreGuard ignoreGuard; + std::unique_lock guard(scheduledEpochMutex_); + auto epoch = scheduleGC_(); + if (auto scheduled = scheduledEpoch_) { + RuntimeAssert( + *scheduled <= epoch, "Scheduled epoch %" PRId64 " which is somehow less previously scheduled %" PRId64, epoch, *scheduled); + } + scheduledEpoch_ = epoch; + return epoch; +} + +Epoch gcScheduler::internal::EpochScheduler::scheduleNextEpochIfNotInProgress() noexcept { + // The locks here are always short-lived, + // so we ignore thread state switching to avoid recursive safe points. + CallsCheckerIgnoreGuard ignoreGuard; + std::unique_lock guard(scheduledEpochMutex_); + if (auto scheduled = scheduledEpoch_) { + return *scheduled; + } + auto epoch = scheduleGC_(); + scheduledEpoch_ = epoch; + return epoch; +} + +void gcScheduler::internal::EpochScheduler::onGCFinish(Epoch epoch) noexcept { + std::unique_lock guard(scheduledEpochMutex_); + RuntimeAssert(scheduledEpoch_ != std::nullopt, "GC for epoch %" PRId64 " happened without going through the GC scheduler", epoch); + auto scheduled = *scheduledEpoch_; + RuntimeAssert( + scheduled >= epoch, "GC for epoch %" PRId64 " happened without going through the GC scheduler, which was waiting for %" PRId64, + epoch, scheduled); + if (scheduled > epoch) + // Waiting for one of the next GC epochs to finish. + return; + // Current GC epoch is finished. + scheduledEpoch_ = std::nullopt; +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.hpp new file mode 100644 index 00000000000..8279cf74a7e --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.hpp @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include +#include + +#include "Utils.hpp" + +namespace kotlin::gcScheduler::internal { + +// Control scheduling new GC epochs. +// TODO: The actual logic is split between this class and `gc::GCState`. The latter +// should be merged into this one. +class EpochScheduler : private Pinned { +public: + using Epoch = int64_t; + + explicit EpochScheduler(std::function scheduleGC) noexcept : scheduleGC_(std::move(scheduleGC)) {} + + // Schedule the next GC epoch unless the GC has a scheduled epoch already. + // If the GC is currently performing collection, this will schedule the next one. + // Returns the currently scheduled GC epoch. + Epoch scheduleNextEpoch() noexcept; + + // Schedule the next GC epoch unless the GC has a scheduled epoch, or is currently + // performing collection. If the GC is currently processing finalizers, this will + // schedule the next GC collection. + // Returns the currently scheduled GC epoch. + Epoch scheduleNextEpochIfNotInProgress() noexcept; + + // Must be called when the GC has finished collection. + void onGCFinish(Epoch epoch) noexcept; + +private: + std::function scheduleGC_; + std::optional scheduledEpoch_; + std::mutex scheduledEpochMutex_; +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochSchedulerTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochSchedulerTest.cpp new file mode 100644 index 00000000000..3237ecaf765 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/EpochSchedulerTest.cpp @@ -0,0 +1,170 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "EpochScheduler.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "TestSupport.hpp" + +using namespace kotlin; + +using gcScheduler::internal::EpochScheduler; +using Epoch = EpochScheduler::Epoch; + +TEST(EpochSchedulerTest, ScheduleNext) { + testing::StrictMock> scheduleGC; + EpochScheduler adapter(scheduleGC.AsStdFunction()); + + // Schedule new epoch. + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 1); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Schedule already scheduled epoch. + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 1); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Schedule new epoch, while the other is still in progress + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 2); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Finish the first epoch. + adapter.onGCFinish(1); + + // Schedule yet another epoch, while the other is still in progress + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(3)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 3); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Finish all epochs + adapter.onGCFinish(2); + adapter.onGCFinish(3); + + // Schedule and finish the final epoch + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(4)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 4); + testing::Mock::VerifyAndClear(&scheduleGC); + adapter.onGCFinish(4); +} + +TEST(EpochSchedulerTest, ScheduleNextIfNotInProgress) { + testing::StrictMock> scheduleGC; + EpochScheduler adapter(scheduleGC.AsStdFunction()); + + // Schedule new epoch. + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1)); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Schedule already scheduled epoch. + EXPECT_CALL(scheduleGC, Call()).Times(0); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Finish the first epoch. + adapter.onGCFinish(1); + + // Schedule and finish the final epoch + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2)); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2); + testing::Mock::VerifyAndClear(&scheduleGC); + adapter.onGCFinish(2); +} + +TEST(EpochSchedulerTest, ScheduleNextMix) { + testing::StrictMock> scheduleGC; + EpochScheduler adapter(scheduleGC.AsStdFunction()); + + // Schedule new epoch. + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1)); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Schedule new epoch, while the other is still in progress + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2)); + EXPECT_THAT(adapter.scheduleNextEpoch(), 2); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Schedule already scheduled epoch. + EXPECT_CALL(scheduleGC, Call()).Times(0); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Finish the first epoch. + adapter.onGCFinish(1); + + // Schedule already scheduled epoch again. + EXPECT_CALL(scheduleGC, Call()).Times(0); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2); + testing::Mock::VerifyAndClear(&scheduleGC); + + // Finish the second epoch. + adapter.onGCFinish(2); + + // Schedule and finish the final epoch + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(3)); + EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 3); + testing::Mock::VerifyAndClear(&scheduleGC); + adapter.onGCFinish(3); +} + +TEST(EpochSchedulerTest, StressScheduleNext) { + constexpr Epoch epochsCount = 1000; + + std::mutex epochsMutex; // Protects scheduled and started relationship. + int64_t scheduledEpoch = 0; + int64_t startedEpoch = 0; + int64_t completedEpoch = 0; + auto scheduleGC = [&]() -> Epoch { + std::unique_lock guard(epochsMutex); + EXPECT_THAT(scheduledEpoch, testing::Ge(startedEpoch)); + if (scheduledEpoch == startedEpoch) { + scheduledEpoch = startedEpoch + 1; + return scheduledEpoch; + } + EXPECT_THAT(scheduledEpoch, startedEpoch + 1); + return scheduledEpoch; + }; + EpochScheduler adapter(std::move(scheduleGC)); + auto startGC = [&]() -> bool { + std::unique_lock guard(epochsMutex); + if (startedEpoch == scheduledEpoch) return false; + EXPECT_THAT(startedEpoch, scheduledEpoch - 1); + startedEpoch = scheduledEpoch; + return true; + }; + auto completeGC = [&] { + adapter.onGCFinish(startedEpoch); + EXPECT_THAT(completedEpoch, startedEpoch - 1); + completedEpoch = startedEpoch; + }; + + std::atomic canStop = false; + std_support::vector threads; + for (int i = 0; i < kDefaultThreadCount; ++i) { + threads.emplace_back([&, i] { + Epoch pastEpoch = 0; + while (!canStop.load(std::memory_order_relaxed)) { + auto epoch = (i % 2 == 0) ? adapter.scheduleNextEpoch() : adapter.scheduleNextEpochIfNotInProgress(); + EXPECT_THAT(epoch, testing::Ge(pastEpoch)); + pastEpoch = epoch; + std::this_thread::yield(); + } + }); + } + + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + while (!startGC()) { + std::this_thread::yield(); + } + std::this_thread::yield(); + completeGC(); + } + canStop.store(true, std::memory_order_relaxed); +}