[K/N] Make GC scheduler less aggressive

^KT-48537

Merge-request: KT-MR-5319
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>
This commit is contained in:
Alexander Shabalin
2022-01-19 19:11:15 +00:00
committed by Space
parent 3e7b1c027f
commit 69bb2587eb
7 changed files with 1146 additions and 123 deletions
@@ -5,6 +5,8 @@
#include "GCScheduler.hpp"
#include <cmath>
#include "CompilerConstants.hpp"
#include "GlobalData.hpp"
#include "KAssert.h"
@@ -17,6 +19,72 @@ using namespace kotlin;
namespace {
class HeapGrowthController {
public:
explicit HeapGrowthController(gc::GCSchedulerConfig& config) noexcept : config_(config) {}
// Called by the mutators.
void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; }
// Called by the GC thread.
void OnPerformFullGC() noexcept { allocatedBytes_ = 0; }
// Called by the GC thread.
void UpdateAliveSetBytes(size_t bytes) noexcept {
lastAliveSetBytes_ = bytes;
if (config_.autoTune.load()) {
double targetHeapBytes = static_cast<double>(bytes) / config_.targetHeapUtilization;
if (!std::isfinite(targetHeapBytes)) {
// This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything.
return;
}
double minHeapBytes = static_cast<double>(config_.minHeapBytes.load());
double maxHeapBytes = static_cast<double>(config_.maxHeapBytes.load());
targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes);
config_.targetHeapBytes = static_cast<int64_t>(targetHeapBytes);
}
}
// Called by the mutators.
bool NeedsGC() const noexcept {
uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load();
uint64_t targetHeapBytes = config_.targetHeapBytes;
return currentHeapBytes >= targetHeapBytes;
}
private:
gc::GCSchedulerConfig& config_;
// Updated by both the mutators and the GC thread.
std::atomic<size_t> allocatedBytes_ = 0;
// Updated by the GC thread, read by the mutators.
std::atomic<size_t> lastAliveSetBytes_ = 0;
};
class RegularIntervalPacer {
public:
using TimePoint = std::chrono::time_point<std::chrono::steady_clock>;
using CurrentTimeProvider = std::function<TimePoint()>;
RegularIntervalPacer(gc::GCSchedulerConfig& config, CurrentTimeProvider currentTimeProvider) noexcept :
config_(config), currentTimeProvider_(currentTimeProvider), lastGC_(currentTimeProvider_()) {}
// Called by the mutators or the timer thread.
bool NeedsGC() const noexcept {
auto currentTimeProvider = currentTimeProvider_();
return currentTimeProvider >= lastGC_.load() + config_.regularGcInterval();
}
// Called by the GC thread.
void OnPerformFullGC() noexcept { lastGC_ = currentTimeProvider_(); }
private:
gc::GCSchedulerConfig& config_;
CurrentTimeProvider currentTimeProvider_;
// Updated by the GC thread, read by the mutators or the timer thread.
std::atomic<TimePoint> lastGC_;
};
class GCEmptySchedulerData : public gc::GCSchedulerData {
void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {}
void OnPerformFullGC() noexcept override {}
@@ -25,74 +93,70 @@ class GCEmptySchedulerData : public gc::GCSchedulerData {
class GCSchedulerDataWithTimer : public gc::GCSchedulerData {
public:
explicit GCSchedulerDataWithTimer(gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
config_(config), scheduleGC_(std::move(scheduleGC)), timer_(std::chrono::microseconds(config_.regularGcIntervalUs), [this]() {
OnTimer();
return std::chrono::microseconds(config_.regularGcIntervalUs);
GCSchedulerDataWithTimer(
gc::GCSchedulerConfig& config,
std::function<void()> scheduleGC,
RegularIntervalPacer::CurrentTimeProvider currentTimeProvider) noexcept :
config_(config),
heapGrowthController_(config),
regularIntervalPacer_(config, currentTimeProvider),
scheduleGC_(std::move(scheduleGC)),
timer_(config_.regularGcInterval(), [this]() {
if (regularIntervalPacer_.NeedsGC()) {
scheduleGC_();
}
return config_.regularGcInterval();
}) {}
void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {
size_t allocatedBytes = threadData.allocatedBytes();
if (allocatedBytes > config_.allocationThresholdBytes) {
RuntimeAssert(static_cast<bool>(scheduleGC_), "scheduleGC_ cannot be empty");
heapGrowthController_.OnAllocated(threadData.allocatedBytes());
if (heapGrowthController_.NeedsGC()) {
scheduleGC_();
}
}
void OnPerformFullGC() noexcept override {}
void UpdateAliveSetBytes(size_t bytes) noexcept override {}
private:
void OnTimer() noexcept {
auto allThreadsAreNative = []() {
auto threadRegistryIter = mm::GlobalData::Instance().threadRegistry().LockForIter();
return std::all_of(threadRegistryIter.begin(), threadRegistryIter.end(), [](mm::ThreadData& thread) {
return thread.state() == ThreadState::kNative;
});
}();
// Don't run, if kotlin code is not being executed.
if (allThreadsAreNative) return;
// TODO: Probably makes sense to check memory usage of the process.
scheduleGC_();
void OnPerformFullGC() noexcept override {
heapGrowthController_.OnPerformFullGC();
regularIntervalPacer_.OnPerformFullGC();
}
gc::GCSchedulerConfig& config_;
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
private:
gc::GCSchedulerConfig& config_;
HeapGrowthController heapGrowthController_;
RegularIntervalPacer regularIntervalPacer_;
std::function<void()> scheduleGC_;
RepeatedTimer timer_;
};
class GCSchedulerDataWithoutTimer : public gc::GCSchedulerData {
class GCSchedulerDataOnSafepoints : public gc::GCSchedulerData {
public:
using CurrentTimeCallback = std::function<uint64_t()>;
GCSchedulerDataWithoutTimer(
gc::GCSchedulerConfig& config, std::function<void()> scheduleGC, CurrentTimeCallback currentTimeCallbackNs) noexcept :
config_(config),
currentTimeCallbackNs_(std::move(currentTimeCallbackNs)),
timeOfLastGcNs_(currentTimeCallbackNs_()),
scheduleGC_(std::move(scheduleGC)) {}
GCSchedulerDataOnSafepoints(
gc::GCSchedulerConfig& config,
std::function<void()> scheduleGC,
RegularIntervalPacer::CurrentTimeProvider currentTimeProvider) noexcept :
heapGrowthController_(config), regularIntervalPacer_(config, currentTimeProvider), scheduleGC_(std::move(scheduleGC)) {}
void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {
size_t allocatedBytes = threadData.allocatedBytes();
if (allocatedBytes > config_.allocationThresholdBytes ||
currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs) {
RuntimeAssert(static_cast<bool>(scheduleGC_), "scheduleGC_ cannot be empty");
heapGrowthController_.OnAllocated(threadData.allocatedBytes());
if (heapGrowthController_.NeedsGC()) {
scheduleGC_();
} else if (regularIntervalPacer_.NeedsGC()) {
scheduleGC_();
}
}
void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); }
void OnPerformFullGC() noexcept override {
heapGrowthController_.OnPerformFullGC();
regularIntervalPacer_.OnPerformFullGC();
}
void UpdateAliveSetBytes(size_t bytes) noexcept override {}
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
private:
gc::GCSchedulerConfig& config_;
CurrentTimeCallback currentTimeCallbackNs_;
std::atomic<uint64_t> timeOfLastGcNs_;
HeapGrowthController heapGrowthController_;
RegularIntervalPacer regularIntervalPacer_;
std::function<void()> scheduleGC_;
};
@@ -115,44 +179,30 @@ private:
std::function<void()> scheduleGC_;
};
KStdUniquePtr<gc::GCSchedulerData> MakeEmptyGCSchedulerData() noexcept {
return ::make_unique<GCEmptySchedulerData>();
}
KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerDataWithTimer(
gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
return ::make_unique<GCSchedulerDataWithTimer>(config, std::move(scheduleGC));
}
KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerDataWithoutTimer(
gc::GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept {
return ::make_unique<GCSchedulerDataWithoutTimer>(config, std::move(scheduleGC), std::move(currentTimeCallbackNs));
}
KStdUniquePtr<gc::GCSchedulerData> MakeGCShedulerDataAggressive(gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
return ::make_unique<GCSchedulerDataAggressive>(config, std::move(scheduleGC));
}
} // namespace
KStdUniquePtr<gc::GCSchedulerData> kotlin::gc::MakeGCSchedulerData(SchedulerType type, gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
KStdUniquePtr<gc::GCSchedulerData> kotlin::gc::internal::MakeGCSchedulerData(
SchedulerType type,
gc::GCSchedulerConfig& config,
std::function<void()> scheduleGC,
std::function<std::chrono::time_point<std::chrono::steady_clock>()> currentTimeProvider) noexcept {
switch (type) {
case SchedulerType::kDisabled:
return MakeEmptyGCSchedulerData();
return ::make_unique<GCEmptySchedulerData>();
case SchedulerType::kWithTimer:
return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC));
return ::make_unique<GCSchedulerDataWithTimer>(config, std::move(scheduleGC), std::move(currentTimeProvider));
case SchedulerType::kOnSafepoints:
return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); });
return ::make_unique<GCSchedulerDataOnSafepoints>(config, std::move(scheduleGC), std::move(currentTimeProvider));
case SchedulerType::kAggressive:
return MakeGCShedulerDataAggressive(config, std::move(scheduleGC));
return ::make_unique<GCSchedulerDataAggressive>(config, std::move(scheduleGC));
}
}
void gc::GCScheduler::SetScheduleGC(std::function<void()> scheduleGC) noexcept {
RuntimeAssert(static_cast<bool>(scheduleGC), "scheduleGC cannot be empty");
RuntimeAssert(!static_cast<bool>(scheduleGC_), "scheduleGC must not have been set");
scheduleGC_ = std::move(scheduleGC);
RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call");
gcData_ = MakeGCSchedulerData(compiler::getGCSchedulerType(), config_, scheduleGC_);
gcData_ = internal::MakeGCSchedulerData(
compiler::getGCSchedulerType(), config_, scheduleGC_, []() { return std::chrono::steady_clock::now(); });
}
@@ -7,6 +7,7 @@
#define RUNTIME_GC_COMMON_GC_SCHEDULER_H
#include <atomic>
#include <chrono>
#include <cinttypes>
#include <cstddef>
#include <functional>
@@ -22,13 +23,28 @@ namespace gc {
using SchedulerType = compiler::GCSchedulerType;
// NOTE: When changing default values, reflect them in GC.kt as well.
struct GCSchedulerConfig {
std::atomic<size_t> threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine).
std::atomic<size_t> allocationThresholdBytes = 10 * 1024 * 1024; // 10MiB by default.
std::atomic<uint64_t> cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default.
std::atomic<bool> autoTune = false;
std::atomic<uint64_t> regularGcIntervalUs = 200 * 1000; // 200 milliseconds by default.
// Only used when useGCTimer() is false. How many regular safepoints will trigger slow path.
std::atomic<int32_t> threshold = 100000;
// How many object bytes a thread must allocate to trigger slow path.
std::atomic<int64_t> allocationThresholdBytes = 10 * 1024;
std::atomic<bool> autoTune = true;
// The target interval between collections when Kotlin code is idle. GC will be triggered
// by timer no sooner than this value and no later than twice this value since the previous collection.
std::atomic<int64_t> regularGcIntervalMicroseconds = 10 * 1000 * 1000;
// How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true.
std::atomic<int64_t> targetHeapBytes = 1024 * 1024;
// The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection
// N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped
// between minHeapBytes and maxHeapBytes.
std::atomic<double> targetHeapUtilization = 0.5;
// The minimum value of targetHeapBytes for autoTune = true
std::atomic<int64_t> minHeapBytes = 1024 * 1024;
// The maximum value of targetHeapBytes for autoTune = true
std::atomic<int64_t> maxHeapBytes = std::numeric_limits<int64_t>::max();
std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); }
};
class GCSchedulerThreadData;
@@ -40,6 +56,10 @@ public:
// Called by different mutator threads.
virtual void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept = 0;
// The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC`
// when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished.
// TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead.
// Always called by the GC thread.
virtual void OnPerformFullGC() noexcept = 0;
@@ -63,11 +83,7 @@ public:
switch (compiler::getGCSchedulerType()) {
case compiler::GCSchedulerType::kOnSafepoints:
case compiler::GCSchedulerType::kAggressive:
safePointsCounter_ += weight;
if (safePointsCounter_ < safePointsCounterThreshold_) {
return;
}
OnSafePointSlowPath();
OnSafePointRegularImpl(weight);
return;
default:
return;
@@ -91,6 +107,16 @@ public:
size_t safePointsCounter() const noexcept { return safePointsCounter_; }
private:
friend class GCSchedulerThreadDataTestApi;
void OnSafePointRegularImpl(size_t weight) noexcept {
safePointsCounter_ += weight;
if (safePointsCounter_ < safePointsCounterThreshold_) {
return;
}
OnSafePointSlowPath();
}
void OnSafePointSlowPath() noexcept {
slowPath_(*this);
ClearCountersAndUpdateThresholds();
@@ -113,7 +139,6 @@ private:
size_t safePointsCounterThreshold_ = 0;
};
class GCScheduler : private Pinned {
public:
GCScheduler() noexcept = default;
@@ -138,10 +163,15 @@ private:
std::function<void()> scheduleGC_;
};
namespace internal {
KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerData(
SchedulerType type,
GCSchedulerConfig& config,
std::function<void()> scheduleGC) noexcept;
std::function<void()> scheduleGC,
std::function<std::chrono::time_point<std::chrono::steady_clock>()> currentTime) noexcept;
}
} // namespace gc
} // namespace kotlin
@@ -0,0 +1,725 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "GCScheduler.hpp"
#include <future>
#include <thread>
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "SingleThreadExecutor.hpp"
#include "TestSupport.hpp"
using namespace kotlin;
using testing::_;
namespace kotlin {
namespace gc {
class GCSchedulerThreadDataTestApi : private Pinned {
public:
explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {}
void OnSafePointRegularImpl(size_t weight) { scheduler_.OnSafePointRegularImpl(weight); }
void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; }
private:
GCSchedulerThreadData& scheduler_;
};
TEST(GCSchedulerThreadDataTest, RegularSafePoint) {
constexpr size_t kWeight = 2;
constexpr size_t kCount = 10;
constexpr size_t kThreshold = kCount * kWeight;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = 1;
config.threshold = kThreshold;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold);
});
schedulerTestApi.OnSafePointRegularImpl(kWeight);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(_)).Times(0);
schedulerTestApi.OnSafePointRegularImpl(kWeight);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), kWeight);
}
TEST(GCSchedulerThreadDataTest, AllocationSafePoint) {
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
config.threshold = 1;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
});
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(_)).Times(0);
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kSize);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
}
TEST(GCSchedulerThreadDataTest, ResetByGC) {
constexpr size_t kWeight = 2;
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kThreshold = kCount * kWeight;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
config.threshold = kThreshold;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight);
EXPECT_CALL(slowPath, Call(_)).Times(0);
scheduler.OnStoppedForGC();
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
}
TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) {
constexpr size_t kWeight = 2;
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kThreshold = kCount * kWeight;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
config.threshold = kThreshold;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
config.allocationThresholdBytes = kAllocationThreshold - kSize;
config.threshold = kThreshold - kWeight;
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
scheduler.OnSafePointAllocation(kSize);
}
scheduler.OnStoppedForGC();
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight);
});
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
});
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
}
TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterRegularSafePoint) {
constexpr size_t kWeight = 2;
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kThreshold = kCount * kWeight;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
config.threshold = kThreshold;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
config.allocationThresholdBytes = kAllocationThreshold - kSize;
config.threshold = kThreshold - kWeight;
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold);
});
schedulerTestApi.OnSafePointRegularImpl(kWeight);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight);
});
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
});
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
}
TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) {
constexpr size_t kWeight = 2;
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kThreshold = kCount * kWeight;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(GCSchedulerThreadData&)> slowPath;
GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
config.threshold = kThreshold;
GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
config.allocationThresholdBytes = kAllocationThreshold - kSize;
config.threshold = kThreshold - kWeight;
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold);
});
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.safePointsCounter(), kThreshold - kWeight);
});
for (size_t i = 0; i < kCount - 1; ++i) {
schedulerTestApi.OnSafePointRegularImpl(kWeight);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
});
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_THAT(scheduler.safePointsCounter(), 0);
}
using TimePoint = std::chrono::time_point<std::chrono::steady_clock>;
class MutatorThread : private Pinned {
public:
MutatorThread(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> slowPath) :
executor_(MakeSingleThreadExecutorWithContext<Context>(
[&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); })) {}
std::future<void> Allocate(size_t bytes) {
return executor_.Execute([&, bytes] {
auto& context = executor_.thread().context();
context.threadDataTestApi.SetAllocatedBytes(bytes);
context.slowPath(context.threadData);
});
}
private:
struct Context {
GCSchedulerThreadData threadData;
GCSchedulerThreadDataTestApi threadDataTestApi;
std::function<void(GCSchedulerThreadData&)> slowPath;
Context(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> slowPath) :
threadData(config, [](GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {}
};
SingleThreadExecutor<ThreadWithContext<Context>> executor_;
};
template <compiler::GCSchedulerType schedulerType, int MutatorCount>
class GCSchedulerDataTestApi {
public:
static constexpr TimePoint initialTime = TimePoint();
explicit GCSchedulerDataTestApi(GCSchedulerConfig& config) {
ON_CALL(currentTime_, Call()).WillByDefault([&]() { return time_.load(); });
scheduler_ = internal::MakeGCSchedulerData(schedulerType, config, scheduleGC_.AsStdFunction(), currentTime_.AsStdFunction());
mutators_.reserve(MutatorCount);
for (int i = 0; i < MutatorCount; ++i) {
mutators_.emplace_back(make_unique<MutatorThread>(
config, [this](GCSchedulerThreadData& threadData) { scheduler_->UpdateFromThreadData(threadData); }));
}
}
std::future<void> Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); }
void OnPerformFullGC() { scheduler_->OnPerformFullGC(); }
void UpdateAliveSetBytes(size_t bytes) { scheduler_->UpdateAliveSetBytes(bytes); }
testing::MockFunction<void()>& scheduleGC() { return scheduleGC_; }
template <typename Duration>
void advance_time(Duration duration) {
auto time = time_.load();
while (true) {
auto newTime = time + std::chrono::duration_cast<TimePoint::duration>(duration);
if (time_.compare_exchange_weak(time, newTime)) {
// TODO: Figure out mocking out RepeatedTimer (or clock underneath it) to avoid sleeping.
std::this_thread::sleep_for(duration);
return;
}
}
}
private:
std::atomic<TimePoint> time_ = initialTime;
KStdVector<KStdUniquePtr<MutatorThread>> mutators_;
testing::MockFunction<void()> scheduleGC_;
testing::NiceMock<testing::MockFunction<TimePoint()>> currentTime_;
KStdUniquePtr<GCSchedulerData> scheduler_;
};
TEST(GCSchedulerDataOnSafePoints, CollectOnTargetHeapReached) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = false;
config.targetHeapBytes = (mutatorsCount + 1) * 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kOnSafepoints, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
KStdVector<std::future<void>> futures;
for (int i = 0; i < mutatorsCount; ++i) {
futures.push_back(schedulerTestApi.Allocate(i, 10));
}
for (auto& future : futures) {
future.get();
}
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, mutatorsCount * 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataOnSafePoints, CollectOnTimeoutReached) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(20)).count();
config.autoTune = false;
config.targetHeapBytes = std::numeric_limits<size_t>::max();
GCSchedulerDataTestApi<compiler::GCSchedulerType::kOnSafepoints, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(10));
KStdVector<std::future<void>> futures;
for (int i = 0; i < mutatorsCount; ++i) {
futures.push_back(schedulerTestApi.Allocate(i, 0));
}
for (auto& future : futures) {
future.get();
}
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.advance_time(std::chrono::milliseconds(15));
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataOnSafePoints, FullTimeoutAfterLastGC) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(20)).count();
config.autoTune = false;
config.targetHeapBytes = 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kOnSafepoints, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.advance_time(std::chrono::milliseconds(10));
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(15));
schedulerTestApi.Allocate(0, 0).get();
// It's now 25 ms since the start, but only 15ms since previous collection.
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.advance_time(std::chrono::milliseconds(10));
schedulerTestApi.Allocate(0, 0).get();
// It's now 25 ms since the previous collection.
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataOnSafePoints, DoNotTuneTargetHeap) {
constexpr int mutatorsCount = 1;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = false;
config.targetHeapBytes = 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kOnSafepoints, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
EXPECT_THAT(config.targetHeapBytes.load(), 10);
}
TEST(GCSchedulerDataOnSafePoints, TuneTargetHeap) {
constexpr int mutatorsCount = 1;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = true;
config.targetHeapBytes = 10;
config.targetHeapUtilization = 0.5;
config.minHeapBytes = 5;
config.maxHeapBytes = 50;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kOnSafepoints, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
EXPECT_THAT(config.targetHeapBytes.load(), 20);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 20.
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(20);
EXPECT_THAT(config.targetHeapBytes.load(), 40);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 60.
schedulerTestApi.Allocate(0, 40).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
// But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// Keeping total heap of 60.
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 40
schedulerTestApi.UpdateAliveSetBytes(40);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 50
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 1
schedulerTestApi.UpdateAliveSetBytes(1);
// But the minimum is set to 5.
EXPECT_THAT(config.targetHeapBytes.load(), 5);
}
TEST(GCSchedulerDataWithTimer, CollectOnTargetHeapReached) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = false;
config.targetHeapBytes = (mutatorsCount + 1) * 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kWithTimer, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
KStdVector<std::future<void>> futures;
for (int i = 0; i < mutatorsCount; ++i) {
futures.push_back(schedulerTestApi.Allocate(i, 10));
}
for (auto& future : futures) {
future.get();
}
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, mutatorsCount * 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataWithTimer, CollectOnTimeoutReached) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(200)).count();
config.autoTune = false;
config.targetHeapBytes = std::numeric_limits<size_t>::max();
GCSchedulerDataTestApi<compiler::GCSchedulerType::kWithTimer, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(100));
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.advance_time(std::chrono::milliseconds(150));
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataWithTimer, FullTimeoutAfterLastGC) {
constexpr int mutatorsCount = kDefaultThreadCount;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::milliseconds(200)).count();
config.autoTune = false;
config.targetHeapBytes = 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kWithTimer, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(100));
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(150));
// It's now 250 ms since the start, but only 150ms since previous collection.
// However, the timer has fired once ~50ms ago.
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.advance_time(std::chrono::milliseconds(100));
// It's now 250 ms since the previous collection, but the timer will fire in ~50ms.
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.advance_time(std::chrono::milliseconds(100));
// 350ms since previous collection, and the timer has fired ~50ms ago.
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
}
TEST(GCSchedulerDataWithTimer, DoNotTuneTargetHeap) {
constexpr int mutatorsCount = 1;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = false;
config.targetHeapBytes = 10;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kWithTimer, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
EXPECT_THAT(config.targetHeapBytes.load(), 10);
}
TEST(GCSchedulerDataWithTimer, TuneTargetHeap) {
constexpr int mutatorsCount = 1;
GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = std::chrono::microseconds(std::chrono::minutes(10)).count();
config.autoTune = true;
config.targetHeapBytes = 10;
config.targetHeapUtilization = 0.5;
config.minHeapBytes = 5;
config.maxHeapBytes = 50;
GCSchedulerDataTestApi<compiler::GCSchedulerType::kWithTimer, mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
EXPECT_THAT(config.targetHeapBytes.load(), 20);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 20.
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(20);
EXPECT_THAT(config.targetHeapBytes.load(), 40);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 60.
schedulerTestApi.Allocate(0, 40).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
// But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// Keeping total heap of 60.
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 40
schedulerTestApi.UpdateAliveSetBytes(40);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
// For a total heap of 50
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 1
schedulerTestApi.UpdateAliveSetBytes(1);
// But the minimum is set to 5.
EXPECT_THAT(config.targetHeapBytes.load(), 5);
}
} // namespace gc
} // namespace kotlin
@@ -105,8 +105,12 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory), gcScheduler_(gcScheduler) {
gcScheduler_.SetScheduleGC([]() {
RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
gSafepointFlag = SafepointFlag::kNeedsGC;
// TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running.
// Investigate, if it's possible to move this logic into the scheduler.
SafepointFlag expectedFlag = SafepointFlag::kNone;
if (gSafepointFlag.compare_exchange_strong(expectedFlag, SafepointFlag::kNeedsGC)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
}
});
}
@@ -2635,10 +2635,8 @@ void startGC() {
}
void setGCThreshold(KInt value) {
RuntimeAssert(value > 0, "Must be handled by the caller");
GC_LOG("setGCThreshold %d\n", value)
if (value <= 0) {
ThrowIllegalArgumentException();
}
initGcThreshold(memoryState, value);
}
@@ -2648,10 +2646,8 @@ KInt getGCThreshold() {
}
void setGCCollectCyclesThreshold(KLong value) {
RuntimeAssert(value > 0, "Must be handled by the caller");
GC_LOG("setGCCollectCyclesThreshold %lld\n", value)
if (value <= 0) {
ThrowIllegalArgumentException();
}
initGcCollectCyclesThreshold(memoryState, value);
}
@@ -2661,10 +2657,8 @@ KInt getGCCollectCyclesThreshold() {
}
void setGCThresholdAllocations(KLong value) {
RuntimeAssert(value > 0, "Must be handled by the caller");
GC_LOG("setGCThresholdAllocations %lld\n", value)
if (value <= 0) {
ThrowIllegalArgumentException();
}
memoryState->allocSinceLastGcThreshold = value;
}
@@ -3615,6 +3609,36 @@ OBJ_GETTER(Kotlin_native_internal_GC_findCycle, KRef, KRef root) {
#endif
}
KLong Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds(KRef) {
return 0;
}
void Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds(KRef, KLong) {}
KLong Kotlin_native_internal_GC_getTargetHeapBytes(KRef) {
return 0;
}
void Kotlin_native_internal_GC_setTargetHeapBytes(KRef, KLong) {}
KDouble Kotlin_native_internal_GC_getTargetHeapUtilization(KRef) {
return 1.0;
}
void Kotlin_native_internal_GC_setTargetHeapUtilization(KRef, KDouble) {}
KLong Kotlin_native_internal_GC_getMaxHeapBytes(KRef) {
return -1;
}
void Kotlin_native_internal_GC_setMaxHeapBytes(KRef, KLong) {}
KLong Kotlin_native_internal_GC_getMinHeapBytes(KRef) {
return 0;
}
void Kotlin_native_internal_GC_setMinHeapBytes(KRef, KLong) {}
RUNTIME_NOTHROW KNativePtr CreateStablePointer(KRef any) {
return createStablePointer(any);
}
@@ -5,6 +5,9 @@
package kotlin.native.internal
import kotlin.time.Duration
import kotlin.time.Duration.Companion.microseconds
/**
* ## Cycle garbage collector interface.
*
@@ -25,12 +28,14 @@ object GC {
/**
* To force garbage collection immediately, unless collector is stopped
* with [stop] operation. Even if GC is suspended, [collect] still triggers collection.
* New MM: trigger new collection and wait for its completion.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_collect")
external fun collect()
/**
* Request global cyclic collector, operation is async and just triggers the collection.
* New MM: unused.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_collectCyclic")
external fun collectCyclic()
@@ -38,18 +43,21 @@ object GC {
/**
* Suspend garbage collection. Release candidates are still collected, but
* GC algorithm is not executed.
* New MM: unused.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_suspend")
external fun suspend()
/**
* Resume garbage collection. Can potentially lead to GC immediately.
* New MM: unused.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_resume")
external fun resume()
/**
* Stop garbage collection. Cyclical garbage is no longer collected.
* New MM: unused.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_stop")
external fun stop()
@@ -57,6 +65,7 @@ object GC {
/**
* Start garbage collection. Cyclical garbage produced while GC was stopped
* cannot be reclaimed, but all new garbage is collected.
* New MM: unused.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_start")
external fun start()
@@ -64,29 +73,59 @@ object GC {
/**
* GC threshold, controlling how frequenly GC is activated, and how much time GC
* takes. Bigger values lead to longer GC pauses, but less GCs.
* New MM: usually unused. For the on-safepoints GC scheduler counts
* how many safepoints must the code pass before informing the GC scheduler.
*
* Default: (old MM) 8 * 1024
* Default: (new MM) 100000
*
* @throws [IllegalArgumentException] when value is not positive.
*/
var threshold: Int
get() = getThreshold()
set(value) = setThreshold(value)
set(value) {
require(value > 0) { "threshold must be positive: $value" }
setThreshold(value)
}
/**
* GC allocation threshold, controlling how frequenly GC collect cycles, and how much time
* this process takes. Bigger values lead to longer GC pauses, but less GCs.
* New MM: unused.
*
* Default: 8 * 1024
*
* @throws [IllegalArgumentException] when value is not positive.
*/
var collectCyclesThreshold: Long
get() = getCollectCyclesThreshold()
set(value) = setCollectCyclesThreshold(value)
set(value) {
require(value > 0) { "collectCyclesThreshold must be positive: $value" }
setCollectCyclesThreshold(value)
}
/**
* GC allocation threshold, controlling how many bytes allocated since last
* collection will trigger new GC.
* New MM: how many bytes a thread can allocate before informing the GC scheduler.
*
* Default: (old MM) 8 * 1024 * 1024
* Default: (new MM) 10 * 1024
*
* @throws [IllegalArgumentException] when value is not positive.
*/
var thresholdAllocations: Long
get() = getThresholdAllocations()
set(value) = setThresholdAllocations(value)
set(value) {
require(value > 0) { "thresholdAllocations must be positive: $value" }
setThresholdAllocations(value)
}
/**
* If GC shall auto-tune thresholds, depending on how much time is spent in collection.
* New MM: if true update targetHeapBytes after each collection.
*
* Default: true
*/
var autotune: Boolean
get() = getTuneThreshold()
@@ -95,21 +134,110 @@ object GC {
/**
* If cyclic collector for atomic references to be deployed.
* New MM: unused.
*/
var cyclicCollectorEnabled: Boolean
get() = getCyclicCollectorEnabled()
set(value) = setCyclicCollectorEnabled(value)
/**
* New MM only. Unused with on-safepoints GC scheduler.
* When Kotlin code is not allocating enough to trigger GC, the GC scheduler uses timer to drive collection.
* Timer-triggered collection will happen roughly in [regularGCInterval] .. 2 * [regularGCInterval] since
* any previous collection.
*
* Default: 10 seconds
*
* @throws [IllegalArgumentException] when value is negative.
*/
var regularGCInterval: Duration
get() = getRegularGCIntervalMicroseconds().microseconds
set(value) {
require(!value.isNegative()) { "regularGCInterval must not be negative: $value" }
setRegularGCIntervalMicroseconds(value.inWholeMicroseconds)
}
/**
* New MM only.
* Total amount of heap available for Kotlin objects. When Kotlin objects overflow this heap,
* the garbage collection is requested. Automatically adjusts when [autotune] is true:
* after each collection the [targetHeapBytes] is set to heapBytes / [targetHeapUtilization] and
* capped between [minHeapBytes] and [maxHeapBytes], where heapBytes is heap usage after the garbage
* is collected.
* Note, that if after a collection heapBytes > [targetHeapBytes] (which may happen if [autotune] is false,
* or [maxHeapBytes] is set too low), the next collection will be triggered almost immediately.
*
* Default: 1 MiB
*
* @throws [IllegalArgumentException] when value is negative.
*/
var targetHeapBytes: Long
get() = getTargetHeapBytes()
set(value) {
require(value >= 0) { "targetHeapBytes must not be negative: $value" }
setTargetHeapBytes(value)
}
/**
* New MM only.
* What fraction of the Kotlin heap should be populated.
* Only used if [autotune] is true. See [targetHeapBytes] for more details.
*
* Default: 0.5
*
* @throws [IllegalArgumentException] when value is outside (0, 1] interval.
*/
var targetHeapUtilization: Double
get() = getTargetHeapUtilization()
set(value) {
require(value > 0 && value <= 1) { "targetHeapUtilization must be in (0, 1] interval: $value" }
setTargetHeapUtilization(value)
}
/**
* New MM only.
* The minimum value for [targetHeapBytes]
* Only used if [autotune] is true. See [targetHeapBytes] for more details.
*
* Default: 1 MiB
*
* @throws [IllegalArgumentException] when value is negative.
*/
var minHeapBytes: Long
get() = getMinHeapBytes()
set(value) {
require(value >= 0) { "minHeapBytes must not be negative: $value" }
setMinHeapBytes(value)
}
/**
* New MM only.
* The maximum value for [targetHeapBytes].
* Only used if [autotune] is true. See [targetHeapBytes] for more details.
*
* Default: [Long.MAX_VALUE]
*
* @throws [IllegalArgumentException] when value is negative.
*/
var maxHeapBytes: Long
get() = getMaxHeapBytes()
set(value) {
require(value >= 0) { "maxHeapBytes must not be negative: $value" }
setMaxHeapBytes(value)
}
/**
* Detect cyclic references going via atomic references and return list of cycle-inducing objects
* or `null` if the leak detector is not available. Use [Platform.isMemoryLeakCheckerActive] to check
* leak detector availability.
* New MM: unused. Always returns null.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_detectCycles")
external fun detectCycles(): Array<Any>?
/**
* Find a reference cycle including from the given object, `null` if no cycles detected.
* New MM: unused. Always returns null.
*/
@GCUnsafeCall("Kotlin_native_internal_GC_findCycle")
external fun findCycle(root: Any): Array<Any>?
@@ -143,4 +271,34 @@ object GC {
@GCUnsafeCall("Kotlin_native_internal_GC_setCyclicCollector")
private external fun setCyclicCollectorEnabled(value: Boolean)
@GCUnsafeCall("Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds")
private external fun getRegularGCIntervalMicroseconds(): Long
@GCUnsafeCall("Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds")
private external fun setRegularGCIntervalMicroseconds(value: Long)
@GCUnsafeCall("Kotlin_native_internal_GC_getTargetHeapBytes")
private external fun getTargetHeapBytes(): Long
@GCUnsafeCall("Kotlin_native_internal_GC_setTargetHeapBytes")
private external fun setTargetHeapBytes(value: Long)
@GCUnsafeCall("Kotlin_native_internal_GC_getTargetHeapUtilization")
private external fun getTargetHeapUtilization(): Double
@GCUnsafeCall("Kotlin_native_internal_GC_setTargetHeapUtilization")
private external fun setTargetHeapUtilization(value: Double)
@GCUnsafeCall("Kotlin_native_internal_GC_getMinHeapBytes")
private external fun getMinHeapBytes(): Long
@GCUnsafeCall("Kotlin_native_internal_GC_setMinHeapBytes")
private external fun setMinHeapBytes(value: Long)
@GCUnsafeCall("Kotlin_native_internal_GC_getMaxHeapBytes")
private external fun getMaxHeapBytes(): Long
@GCUnsafeCall("Kotlin_native_internal_GC_setMaxHeapBytes")
private external fun setMaxHeapBytes(value: Long)
}
+59 -27
View File
@@ -299,7 +299,7 @@ extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) {
extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) {
// TODO: Remove when legacy MM is gone.
ThrowIllegalArgumentException();
// Nothing to do
}
// TODO: Maybe a pair of suspend/resume or start/stop may be useful in the future?
@@ -321,46 +321,33 @@ extern "C" void Kotlin_native_internal_GC_start(ObjHeader*) {
// Nothing to do
}
extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, int32_t value) {
if (value < 0) {
ThrowIllegalArgumentException();
}
mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = static_cast<size_t>(value);
extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, KInt value) {
RuntimeAssert(value > 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = value;
}
extern "C" int32_t Kotlin_native_internal_GC_getThreshold(ObjHeader*) {
auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load();
auto maxValue = std::numeric_limits<int32_t>::max();
if (threshold > static_cast<size_t>(maxValue)) {
return maxValue;
}
return static_cast<int32_t>(maxValue);
extern "C" KInt Kotlin_native_internal_GC_getThreshold(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load();
}
extern "C" void Kotlin_native_internal_GC_setCollectCyclesThreshold(ObjHeader*, int64_t value) {
// TODO: Remove when legacy MM is gone.
ThrowIllegalArgumentException();
// Nothing to do
}
extern "C" int64_t Kotlin_native_internal_GC_getCollectCyclesThreshold(ObjHeader*) {
// TODO: Remove when legacy MM is gone.
ThrowIllegalArgumentException();
// Nothing to do
return -1;
}
extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, int64_t value) {
if (value < 0) {
ThrowIllegalArgumentException();
}
mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = static_cast<size_t>(value);
RuntimeAssert(value > 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = value;
}
extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) {
auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load();
auto maxValue = std::numeric_limits<int64_t>::max();
if (threshold > static_cast<size_t>(maxValue)) {
return maxValue;
}
return static_cast<int64_t>(maxValue);
return mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load();
}
extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) {
@@ -371,6 +358,51 @@ extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune.load();
}
extern "C" KLong Kotlin_native_internal_GC_getRegularGCIntervalMicroseconds(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds.load();
}
extern "C" void Kotlin_native_internal_GC_setRegularGCIntervalMicroseconds(ObjHeader*, KLong value) {
RuntimeAssert(value >= 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().regularGcIntervalMicroseconds = value;
}
extern "C" KLong Kotlin_native_internal_GC_getTargetHeapBytes(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes.load();
}
extern "C" void Kotlin_native_internal_GC_setTargetHeapBytes(ObjHeader*, KLong value) {
RuntimeAssert(value >= 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapBytes = value;
}
extern "C" KDouble Kotlin_native_internal_GC_getTargetHeapUtilization(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization.load();
}
extern "C" void Kotlin_native_internal_GC_setTargetHeapUtilization(ObjHeader*, KDouble value) {
RuntimeAssert(value > 0 && value <= 1, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().targetHeapUtilization = value;
}
extern "C" KLong Kotlin_native_internal_GC_getMaxHeapBytes(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes.load();
}
extern "C" void Kotlin_native_internal_GC_setMaxHeapBytes(ObjHeader*, KLong value) {
RuntimeAssert(value >= 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().maxHeapBytes = value;
}
extern "C" KLong Kotlin_native_internal_GC_getMinHeapBytes(ObjHeader*) {
return mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes.load();
}
extern "C" void Kotlin_native_internal_GC_setMinHeapBytes(ObjHeader*, KLong value) {
RuntimeAssert(value >= 0, "Must be handled by the caller");
mm::GlobalData::Instance().gc().gcSchedulerConfig().minHeapBytes = value;
}
extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) {
// TODO: Remove when legacy MM is gone.
RETURN_OBJ(nullptr);
@@ -383,13 +415,13 @@ extern "C" OBJ_GETTER(Kotlin_native_internal_GC_findCycle, ObjHeader*, ObjHeader
extern "C" bool Kotlin_native_internal_GC_getCyclicCollector(ObjHeader* gc) {
// TODO: Remove when legacy MM is gone.
// Nothing to do.
return false;
}
extern "C" void Kotlin_native_internal_GC_setCyclicCollector(ObjHeader* gc, bool value) {
// TODO: Remove when legacy MM is gone.
if (value)
ThrowIllegalArgumentException();
// Nothing to do.
}
extern "C" bool Kotlin_Any_isShareable(ObjHeader* thiz) {