From fc13dbdb913e2561545e7916949d9e6dc0a38eb6 Mon Sep 17 00:00:00 2001 From: Troels Bjerre Lund Date: Fri, 22 Sep 2023 15:31:28 +0200 Subject: [PATCH] [K/N] Track live size on sweep After the change to track allocations in big chunks (KT-57773), the scheduler no longer had information about how much of the allocated memory was used on live objects, which lead to a lot of extra memory usage (KT-61914). This patch adds the tracking of live objects size to sweeping, which is enough to mitigate the problem. co-authored by Alexander Shabalin ^KT-61914 --- .../runtime/src/alloc/custom/cpp/GCApi.cpp | 9 +++++---- .../src/alloc/legacy/cpp/ObjectFactorySweep.hpp | 8 ++++---- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 5 +---- .../runtime/src/gc/common/cpp/GCStatistics.cpp | 17 +++++++++++++++++ .../runtime/src/gc/common/cpp/GCStatistics.hpp | 13 ++++++++++--- .../src/gc/stms/cpp/SameThreadMarkAndSweep.cpp | 8 +------- .../common/cpp/GCSchedulerConfig.hpp | 10 ++-------- 7 files changed, 40 insertions(+), 30 deletions(-) diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp index 54d83bdef6a..07e992cf4f6 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp @@ -16,10 +16,10 @@ #endif #include "CompilerConstants.hpp" +#include "CustomAllocator.hpp" #include "CustomLogging.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" -#include "FinalizerHooks.hpp" #include "GC.hpp" #include "GCStatistics.hpp" #include "KAssert.h" @@ -35,9 +35,10 @@ namespace kotlin::alloc { bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { auto* heapObjHeader = reinterpret_cast(object); + auto size = CustomAllocator::GetAllocatedHeapSize(heapObjHeader->object()); if (gc::tryResetMark(heapObjHeader->objectData())) { CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader); - gcHandle.addKeptObject(); + gcHandle.addKeptObject(size); return true; } auto* extraObject = mm::ExtraObjectData::Get(heapObjHeader->object()); @@ -49,13 +50,13 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle:: CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject)); finalizerQueue.Push(ExtraObjectCell::fromExtraObject(extraObject)); gcHandle.addMarkedObject(); - gcHandle.addKeptObject(); + gcHandle.addKeptObject(size); return true; } if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { CustomAllocDebug("SweepObject(%p): already waiting to be finalized", heapObjHeader); gcHandle.addMarkedObject(); - gcHandle.addKeptObject(); + gcHandle.addKeptObject(size); return true; } extraObject->UnlinkFromBaseObject(); diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp index 53c4a359930..d8d26f4dc6e 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp @@ -23,7 +23,7 @@ void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory if (extraObject.HasAssociatedObject()) { extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); ++it; - sweepHandle.addKeptObject(); + sweepHandle.addKeptObject(sizeof(mm::ExtraObjectData)); } else { extraObject.Uninstall(); it.EraseAndAdvance(); @@ -31,7 +31,7 @@ void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory } } else { ++it; - sweepHandle.addKeptObject(); + sweepHandle.addKeptObject(sizeof(mm::ExtraObjectData)); } } } @@ -48,13 +48,13 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typena auto sweepHandle = handle.sweep(); for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) { + auto* objHeader = it->GetObjHeader(); if (Traits::TryResetMark(*it)) { ++it; - sweepHandle.addKeptObject(); + sweepHandle.addKeptObject(Traits::ObjectFactory::GetAllocatedHeapSize(objHeader)); continue; } sweepHandle.addSweptObject(); - auto* objHeader = it->GetObjHeader(); if (HasFinalizers(objHeader)) { objectFactoryIter.MoveAndAdvance(finalizerQueue, it); } else { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 27c1bef79a5..d3e0b990866 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -5,14 +5,11 @@ #include "ConcurrentMarkAndSweep.hpp" -#include #include #include "AllocatorImpl.hpp" #include "CallsChecker.hpp" #include "CompilerConstants.hpp" -#include "GlobalData.hpp" -#include "GCImpl.hpp" #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" #include "Memory.h" @@ -209,7 +206,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { } finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, alloc::allocatedBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index 702e9d018c6..bef8339aa5e 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -379,6 +379,21 @@ MarkStats GCHandle::getMarked() { return MarkStats{}; } +size_t GCHandle::getKeptSizeBytes() noexcept { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + size_t result = 0; + if (stat->sweepStats.heap) { + result += stat->sweepStats.heap->keptSizeBytes; + } + if (stat->sweepStats.extra) { + result += stat->sweepStats.extra->keptSizeBytes; + } + return result; + } + return 0; +} + void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { @@ -388,6 +403,7 @@ void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept { } heap->keptCount += stats.keptCount; heap->sweptCount += stats.sweptCount; + heap->keptSizeBytes += stats.keptSizeBytes; RuntimeAssert(static_cast(stat->markStats), "Mark must have already happened"); stat->markStats->markedCount += markedCount; } @@ -402,6 +418,7 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept { } extra->keptCount += stats.keptCount; extra->sweptCount += stats.sweptCount; + extra->keptSizeBytes += stats.keptSizeBytes; } } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 01c0c6012f0..99b384f5097 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -8,7 +8,6 @@ #include #include -#include "Common.h" #include "Logging.hpp" #include "Porting.h" #include "Utils.hpp" @@ -28,6 +27,7 @@ class GCHandle; struct SweepStats { uint64_t sweptCount = 0; uint64_t keptCount = 0; + size_t keptSizeBytes = 0; }; struct MarkStats { @@ -69,6 +69,7 @@ public: [[nodiscard]] GCProcessWeaksScope processWeaks() noexcept; MarkStats getMarked(); + size_t getKeptSizeBytes() noexcept; private: uint64_t epoch_; @@ -117,7 +118,10 @@ public: ~GCSweepScope(); void addSweptObject() noexcept { stats_.sweptCount += 1; } - void addKeptObject() noexcept { stats_.keptCount += 1; } + void addKeptObject(size_t sizeBytes) noexcept { + stats_.keptCount += 1; + stats_.keptSizeBytes += sizeBytes; + } // Custom allocator only. To be finalized objects are kept alive. void addMarkedObject() noexcept { markedCount_ += 1; } }; @@ -132,7 +136,10 @@ public: ~GCSweepExtraObjectsScope(); void addSweptObject() noexcept { stats_.sweptCount += 1; } - void addKeptObject() noexcept { stats_.keptCount += 1; } + void addKeptObject(size_t sizeBytes) noexcept { + stats_.keptCount += 1; + stats_.keptSizeBytes += sizeBytes; + } }; class GCHandle::GCGlobalRootSetScope : private GCStageScopeBase { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 012d0ebaa03..d1e235d989a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -5,17 +5,11 @@ #include "SameThreadMarkAndSweep.hpp" -#include - -#include "CompilerConstants.hpp" -#include "GlobalData.hpp" -#include "GCImpl.hpp" #include "GCStatistics.hpp" #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" #include "Memory.h" #include "RootSet.hpp" -#include "Runtime.h" #include "ThreadData.hpp" #include "ThreadRegistry.hpp" #include "ThreadSuspension.hpp" @@ -102,7 +96,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, alloc::allocatedBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); resumeTheWorld(gcHandle); diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp index 83e06dce2ad..6d236210ba4 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp @@ -11,12 +11,6 @@ namespace kotlin::gcScheduler { -#if KONAN_WATCHOS -constexpr int64_t kDefaultTargetHeapBytes = 10 * 1024 * 1024; -#else -constexpr int64_t kDefaultTargetHeapBytes = 100 * 1024 * 1024; -#endif - // NOTE: When changing default values, reflect them in GC.kt as well. struct GCSchedulerConfig { enum class MutatorAssists { @@ -33,7 +27,7 @@ struct GCSchedulerConfig { // become bigger than this value, and `mutatorAssists` are enabled the GC will // stop the world and wait until current epoch finishes. // Adapts after each GC epoch when `autoTune = true`. - std::atomic targetHeapBytes = kDefaultTargetHeapBytes; + std::atomic targetHeapBytes = 10 * 1024 * 1024; // The rate at which `targetHeapBytes` changes when `autoTune = true`. Concretely: if after the collection // `N` object bytes remain in the heap, the next `targetHeapBytes` will be `N / targetHeapUtilization` capped // between `minHeapBytes` and `maxHeapBytes`. @@ -46,7 +40,7 @@ struct GCSchedulerConfig { std::atomic heapTriggerCoefficient = 0.9; // See `mutatorAssists()`. std::atomic> mutatorAssistsImpl = - static_cast>(MutatorAssists::kDefault); + static_cast>(MutatorAssists::kDisable); std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); }