[K/N] Make GC scheduler pausing less aggressive ^KT-61091

kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp

@@ -5,7 +5,6 @@
 
 #include "GCSchedulerImpl.hpp"
 
-#include "CallsChecker.hpp"
 #include "GlobalData.hpp"
 #include "Memory.h"
 #include "Logging.hpp"
@@ -23,9 +22,6 @@ gcScheduler::GCScheduler::ThreadData::~ThreadData() = default;
 
 gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept :
     impl_(config, []() noexcept {
-        // This call acquires a lock, but the lock are always short-lived,
-        // so we ignore thread state switching to avoid recursive safe points.
-        CallsCheckerIgnoreGuard guard;
         return mm::GlobalData::Instance().gc().Schedule();
     }) {}
 

kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp

@@ -16,6 +16,7 @@
 #include "RegularIntervalPacer.hpp"
 #include "RepeatedTimer.hpp"
 #include "SafePoint.hpp"
+#include "EpochScheduler.hpp"
 #include "ThreadData.hpp"
 
 namespace kotlin::gcScheduler {
@@ -55,7 +56,7 @@ public:
             }
             if (regularIntervalPacer_.NeedsGC()) {
                 RuntimeLogDebug({kTagGC}, "Scheduling GC by timer");
-                schedule();
+                scheduleGC_.scheduleNextEpochIfNotInProgress();
             }
         }) {
         RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized");
@@ -73,18 +74,19 @@ public:
                 return;
             case HeapGrowthController::MemoryBoundary::kTrigger:
                 RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
-                schedule();
+                scheduleGC_.scheduleNextEpochIfNotInProgress();
                 return;
             case HeapGrowthController::MemoryBoundary::kTarget:
                 RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
-                auto epoch = schedule();
+                auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress();
-                RuntimeLogWarning({kTagGC}, "Pausing the mutators");
+                RuntimeLogWarning({kTagGC}, "Pausing the mutators until epoch %" PRId64 " is done", epoch);
                 mutatorAssists_.requestAssists(epoch);
                 return;
         }
     }
 
     void onGCFinish(int64_t epoch, size_t bytes) noexcept {
+        scheduleGC_.onGCFinish(epoch);
         heapGrowthController_.updateBoundaries(bytes);
         // Must wait for all mutators to be released. GC thread cannot continue.
         // This is the contract between GC and mutators. With regular native state
@@ -95,13 +97,13 @@ public:
         });
     }
 
-    int64_t schedule() noexcept { return scheduleGC_(); }
+    int64_t schedule() noexcept { return scheduleGC_.scheduleNextEpoch(); }
 
     MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; }
 
 private:
     GCSchedulerConfig& config_;
-    std::function<int64_t()> scheduleGC_;
+    EpochScheduler scheduleGC_;
     mm::AppStateTracking& appStateTracking_;
     HeapGrowthController heapGrowthController_;
     RegularIntervalPacer<Clock> regularIntervalPacer_;

kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp

@@ -176,7 +176,7 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) {
     // Wait until the timer is initialized.
     test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
     schedulerTestApi.advance_time(microseconds(10));
     test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
@@ -197,7 +197,7 @@ TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) {
     test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
 
     schedulerTestApi.advance_time(microseconds(5));
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
     schedulerTestApi.Allocate(0, 10).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
     schedulerTestApi.OnPerformFullGC();
@@ -218,7 +218,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotTuneTargetHeap) {
     config.targetHeapBytes = 10;
     GCSchedulerDataTestApi<mutatorsCount> schedulerTestApi(config);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
     schedulerTestApi.Allocate(0, 10).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
     schedulerTestApi.OnPerformFullGC();
@@ -239,7 +239,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
     config.maxHeapBytes = 50;
     GCSchedulerDataTestApi<mutatorsCount> schedulerTestApi(config);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
     schedulerTestApi.Allocate(0, 10).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
     schedulerTestApi.OnPerformFullGC();
@@ -247,7 +247,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
 
     EXPECT_THAT(config.targetHeapBytes.load(), 20);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2));
     // For a total heap of 20.
     schedulerTestApi.Allocate(0, 10).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
@@ -256,7 +256,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
 
     EXPECT_THAT(config.targetHeapBytes.load(), 40);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(3));
     // For a total heap of 60.
     schedulerTestApi.Allocate(0, 40).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
@@ -266,7 +266,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
     // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down
     EXPECT_THAT(config.targetHeapBytes.load(), 50);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(4));
     // Keeping total heap of 60.
     schedulerTestApi.Allocate(0, 0).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
@@ -275,7 +275,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
 
     EXPECT_THAT(config.targetHeapBytes.load(), 50);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(5));
     schedulerTestApi.Allocate(0, 0).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
     schedulerTestApi.OnPerformFullGC();
@@ -284,7 +284,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
 
     EXPECT_THAT(config.targetHeapBytes.load(), 50);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(6));
     // For a total heap of 50
     schedulerTestApi.Allocate(0, 10).get();
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
@@ -324,7 +324,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotCollectOnTimerInBackground) {
     // Now go back into the foreground.
     appStateTracking.setState(mm::AppStateTracking::State::kForeground);
 
-    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
+    EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
     schedulerTestApi.advance_time(microseconds(10));
     test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
     testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());

kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp

@@ -5,7 +5,6 @@
 
 #include "GCSchedulerImpl.hpp"
 
-#include "CallsChecker.hpp"
 #include "GlobalData.hpp"
 #include "Memory.h"
 #include "Logging.hpp"
@@ -23,9 +22,6 @@ gcScheduler::GCScheduler::ThreadData::~ThreadData() = default;
 
 gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept :
     impl_(config, []() noexcept {
-        // This call acquires a lock, but the lock are always short-lived,
-        // so we ignore thread state switching to avoid recursive safe points.
-        CallsCheckerIgnoreGuard guard;
         return mm::GlobalData::Instance().gc().Schedule();
     }) {}
 

kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp

@@ -15,6 +15,7 @@
 #include "MutatorAssists.hpp"
 #include "SafePoint.hpp"
 #include "SafePointTracker.hpp"
+#include "EpochScheduler.hpp"
 #include "ThreadData.hpp"
 
 namespace kotlin::gcScheduler {
@@ -56,11 +57,11 @@ public:
                 return;
             case HeapGrowthController::MemoryBoundary::kTrigger:
                 RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
-                schedule();
+                scheduleGC_.scheduleNextEpochIfNotInProgress();
                 return;
             case HeapGrowthController::MemoryBoundary::kTarget:
                 RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
-                auto epoch = schedule();
+                auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress();
                 RuntimeLogWarning({kTagGC}, "Pausing the mutators");
                 mutatorAssists_.requestAssists(epoch);
                 return;
@@ -75,6 +76,7 @@ public:
     }
 
     void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {
+        scheduleGC_.onGCFinish(epoch);
         heapGrowthController_.updateBoundaries(aliveBytes);
         // Must wait for all mutators to be released. GC thread cannot continue.
         // This is the contract between GC and mutators. With regular native state
@@ -85,12 +87,12 @@ public:
         });
     }
 
-    int64_t schedule() noexcept { return scheduleGC_(); }
+    int64_t schedule() noexcept { return scheduleGC_.scheduleNextEpoch(); }
 
     MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; }
 
 private:
-    std::function<int64_t()> scheduleGC_;
+    EpochScheduler scheduleGC_;
     HeapGrowthController heapGrowthController_;
     SafePointTracker<> safePointTracker_;
     mm::SafePointActivator safePointActivator_;

kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp

@@ -56,25 +56,36 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) {
         gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction());
 
         int i = 0;
+        std::optional<int64_t> scheduled;
         // We trigger GC on the first iteration, when the unique allocation point is faced,
         // on the second to last iteration when weak target heap size is reached,
         // and on the last iteration when target heap size is reached.
         EXPECT_CALL(scheduleGC, Call())
-                .WillOnce([&i]() {
+                .WillOnce([&]() {
                     EXPECT_THAT(i, 0);
-                    return 0;
+                    EXPECT_THAT(scheduled, std::nullopt);
-                })
+                    scheduled = 1;
-                .WillOnce([&i]() {
-                    EXPECT_THAT(i, 8);
                     return 1;
                 })
-                .WillOnce([&i]() {
+                .WillOnce([&]() {
-                    EXPECT_THAT(i, 9);
+                    EXPECT_THAT(i, 8);
+                    EXPECT_THAT(scheduled, std::nullopt);
+                    scheduled = 2;
                     return 2;
+                })
+                .WillOnce([&]() {
+                    EXPECT_THAT(i, 9);
+                    EXPECT_THAT(scheduled, std::nullopt);
+                    scheduled = 3;
+                    return 3;
                 });
 
         for (; i < 10; i++) {
             scheduler.setAllocatedBytes(i + 1);
+            if (scheduled) {
+                scheduler.onGCFinish(*scheduled, i + 1);
+                scheduled = std::nullopt;
+            }
         }
         testing::Mock::VerifyAndClearExpectations(&scheduleGC);
     }();

kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "EpochScheduler.hpp"
+
+#include <cinttypes>
+
+#include "CallsChecker.hpp"
+#include "KAssert.h"
+
+using namespace kotlin;
+
+using Epoch = gcScheduler::internal::EpochScheduler::Epoch;
+
+Epoch gcScheduler::internal::EpochScheduler::scheduleNextEpoch() noexcept {
+    // The locks here are always short-lived,
+    // so we ignore thread state switching to avoid recursive safe points.
+    CallsCheckerIgnoreGuard ignoreGuard;
+    std::unique_lock guard(scheduledEpochMutex_);
+    auto epoch = scheduleGC_();
+    if (auto scheduled = scheduledEpoch_) {
+        RuntimeAssert(
+                *scheduled <= epoch, "Scheduled epoch %" PRId64 " which is somehow less previously scheduled %" PRId64, epoch, *scheduled);
+    }
+    scheduledEpoch_ = epoch;
+    return epoch;
+}
+
+Epoch gcScheduler::internal::EpochScheduler::scheduleNextEpochIfNotInProgress() noexcept {
+    // The locks here are always short-lived,
+    // so we ignore thread state switching to avoid recursive safe points.
+    CallsCheckerIgnoreGuard ignoreGuard;
+    std::unique_lock guard(scheduledEpochMutex_);
+    if (auto scheduled = scheduledEpoch_) {
+        return *scheduled;
+    }
+    auto epoch = scheduleGC_();
+    scheduledEpoch_ = epoch;
+    return epoch;
+}
+
+void gcScheduler::internal::EpochScheduler::onGCFinish(Epoch epoch) noexcept {
+    std::unique_lock guard(scheduledEpochMutex_);
+    RuntimeAssert(scheduledEpoch_ != std::nullopt, "GC for epoch %" PRId64 " happened without going through the GC scheduler", epoch);
+    auto scheduled = *scheduledEpoch_;
+    RuntimeAssert(
+            scheduled >= epoch, "GC for epoch %" PRId64 " happened without going through the GC scheduler, which was waiting for %" PRId64,
+            epoch, scheduled);
+    if (scheduled > epoch)
+        // Waiting for one of the next GC epochs to finish.
+        return;
+    // Current GC epoch is finished.
+    scheduledEpoch_ = std::nullopt;
+}

kotlin-native/runtime/src/gcScheduler/common/cpp/EpochScheduler.hpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include <functional>
+#include <mutex>
+#include <optional>
+
+#include "Utils.hpp"
+
+namespace kotlin::gcScheduler::internal {
+
+// Control scheduling new GC epochs.
+// TODO: The actual logic is split between this class and `gc::GCState`. The latter
+//       should be merged into this one.
+class EpochScheduler : private Pinned {
+public:
+    using Epoch = int64_t;
+
+    explicit EpochScheduler(std::function<Epoch()> scheduleGC) noexcept : scheduleGC_(std::move(scheduleGC)) {}
+
+    // Schedule the next GC epoch unless the GC has a scheduled epoch already.
+    // If the GC is currently performing collection, this will schedule the next one.
+    // Returns the currently scheduled GC epoch.
+    Epoch scheduleNextEpoch() noexcept;
+
+    // Schedule the next GC epoch unless the GC has a scheduled epoch, or is currently
+    // performing collection. If the GC is currently processing finalizers, this will
+    // schedule the next GC collection.
+    // Returns the currently scheduled GC epoch.
+    Epoch scheduleNextEpochIfNotInProgress() noexcept;
+
+    // Must be called when the GC has finished collection.
+    void onGCFinish(Epoch epoch) noexcept;
+
+private:
+    std::function<Epoch()> scheduleGC_;
+    std::optional<int64_t> scheduledEpoch_;
+    std::mutex scheduledEpochMutex_;
+};
+
+} // namespace kotlin::gcScheduler::internal

kotlin-native/runtime/src/gcScheduler/common/cpp/EpochSchedulerTest.cpp

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "EpochScheduler.hpp"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "TestSupport.hpp"
+
+using namespace kotlin;
+
+using gcScheduler::internal::EpochScheduler;
+using Epoch = EpochScheduler::Epoch;
+
+TEST(EpochSchedulerTest, ScheduleNext) {
+    testing::StrictMock<testing::MockFunction<Epoch()>> scheduleGC;
+    EpochScheduler adapter(scheduleGC.AsStdFunction());
+
+    // Schedule new epoch.
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 1);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Schedule already scheduled epoch.
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 1);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Schedule new epoch, while the other is still in progress
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 2);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Finish the first epoch.
+    adapter.onGCFinish(1);
+
+    // Schedule yet another epoch, while the other is still in progress
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(3));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 3);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Finish all epochs
+    adapter.onGCFinish(2);
+    adapter.onGCFinish(3);
+
+    // Schedule and finish the final epoch
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(4));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 4);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+    adapter.onGCFinish(4);
+}
+
+TEST(EpochSchedulerTest, ScheduleNextIfNotInProgress) {
+    testing::StrictMock<testing::MockFunction<Epoch()>> scheduleGC;
+    EpochScheduler adapter(scheduleGC.AsStdFunction());
+
+    // Schedule new epoch.
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1));
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Schedule already scheduled epoch.
+    EXPECT_CALL(scheduleGC, Call()).Times(0);
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Finish the first epoch.
+    adapter.onGCFinish(1);
+
+    // Schedule and finish the final epoch
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2));
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+    adapter.onGCFinish(2);
+}
+
+TEST(EpochSchedulerTest, ScheduleNextMix) {
+    testing::StrictMock<testing::MockFunction<Epoch()>> scheduleGC;
+    EpochScheduler adapter(scheduleGC.AsStdFunction());
+
+    // Schedule new epoch.
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1));
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 1);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Schedule new epoch, while the other is still in progress
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(2));
+    EXPECT_THAT(adapter.scheduleNextEpoch(), 2);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Schedule already scheduled epoch.
+    EXPECT_CALL(scheduleGC, Call()).Times(0);
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Finish the first epoch.
+    adapter.onGCFinish(1);
+
+    // Schedule already scheduled epoch again.
+    EXPECT_CALL(scheduleGC, Call()).Times(0);
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 2);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+
+    // Finish the second epoch.
+    adapter.onGCFinish(2);
+
+    // Schedule and finish the final epoch
+    EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(3));
+    EXPECT_THAT(adapter.scheduleNextEpochIfNotInProgress(), 3);
+    testing::Mock::VerifyAndClear(&scheduleGC);
+    adapter.onGCFinish(3);
+}
+
+TEST(EpochSchedulerTest, StressScheduleNext) {
+    constexpr Epoch epochsCount = 1000;
+
+    std::mutex epochsMutex; // Protects scheduled and started relationship.
+    int64_t scheduledEpoch = 0;
+    int64_t startedEpoch = 0;
+    int64_t completedEpoch = 0;
+    auto scheduleGC = [&]() -> Epoch {
+        std::unique_lock guard(epochsMutex);
+        EXPECT_THAT(scheduledEpoch, testing::Ge(startedEpoch));
+        if (scheduledEpoch == startedEpoch) {
+            scheduledEpoch = startedEpoch + 1;
+            return scheduledEpoch;
+        }
+        EXPECT_THAT(scheduledEpoch, startedEpoch + 1);
+        return scheduledEpoch;
+    };
+    EpochScheduler adapter(std::move(scheduleGC));
+    auto startGC = [&]() -> bool {
+        std::unique_lock guard(epochsMutex);
+        if (startedEpoch == scheduledEpoch) return false;
+        EXPECT_THAT(startedEpoch, scheduledEpoch - 1);
+        startedEpoch = scheduledEpoch;
+        return true;
+    };
+    auto completeGC = [&] {
+        adapter.onGCFinish(startedEpoch);
+        EXPECT_THAT(completedEpoch, startedEpoch - 1);
+        completedEpoch = startedEpoch;
+    };
+
+    std::atomic<bool> canStop = false;
+    std_support::vector<ScopedThread> threads;
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        threads.emplace_back([&, i] {
+            Epoch pastEpoch = 0;
+            while (!canStop.load(std::memory_order_relaxed)) {
+                auto epoch = (i % 2 == 0) ? adapter.scheduleNextEpoch() : adapter.scheduleNextEpochIfNotInProgress();
+                EXPECT_THAT(epoch, testing::Ge(pastEpoch));
+                pastEpoch = epoch;
+                std::this_thread::yield();
+            }
+        });
+    }
+
+    for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
+        while (!startGC()) {
+            std::this_thread::yield();
+        }
+        std::this_thread::yield();
+        completeGC();
+    }
+    canStop.store(true, std::memory_order_relaxed);
+}