[K/N] Track live size on sweep

After the change to track allocations in big chunks (KT-57773), the
scheduler no longer had information about how much of the allocated
memory was used on live objects, which lead to a lot of extra memory
usage (KT-61914). This patch adds the tracking of live objects size to
sweeping, which is enough to mitigate the problem.

co-authored by Alexander Shabalin

^KT-61914

kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp

@@ -16,10 +16,10 @@
 #endif
 
 #include "CompilerConstants.hpp"
+#include "CustomAllocator.hpp"
 #include "CustomLogging.hpp"
 #include "ExtraObjectData.hpp"
 #include "ExtraObjectPage.hpp"
-#include "FinalizerHooks.hpp"
 #include "GC.hpp"
 #include "GCStatistics.hpp"
 #include "KAssert.h"
@@ -35,9 +35,10 @@ namespace kotlin::alloc {
 
 bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
     auto* heapObjHeader = reinterpret_cast<HeapObjHeader*>(object);
+    auto size = CustomAllocator::GetAllocatedHeapSize(heapObjHeader->object());
     if (gc::tryResetMark(heapObjHeader->objectData())) {
         CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader);
-        gcHandle.addKeptObject();
+        gcHandle.addKeptObject(size);
         return true;
     }
     auto* extraObject = mm::ExtraObjectData::Get(heapObjHeader->object());
@@ -49,13 +50,13 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::
             CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject));
             finalizerQueue.Push(ExtraObjectCell::fromExtraObject(extraObject));
             gcHandle.addMarkedObject();
-            gcHandle.addKeptObject();
+            gcHandle.addKeptObject(size);
             return true;
         }
         if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
             CustomAllocDebug("SweepObject(%p): already waiting to be finalized", heapObjHeader);
             gcHandle.addMarkedObject();
-            gcHandle.addKeptObject();
+            gcHandle.addKeptObject(size);
             return true;
         }
         extraObject->UnlinkFromBaseObject();

kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp

@@ -23,7 +23,7 @@ void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory
             if (extraObject.HasAssociatedObject()) {
                 extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
                 ++it;
-                sweepHandle.addKeptObject();
+                sweepHandle.addKeptObject(sizeof(mm::ExtraObjectData));
             } else {
                 extraObject.Uninstall();
                 it.EraseAndAdvance();
@@ -31,7 +31,7 @@ void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory
             }
         } else {
             ++it;
-            sweepHandle.addKeptObject();
+            sweepHandle.addKeptObject(sizeof(mm::ExtraObjectData));
         }
     }
 }
@@ -48,13 +48,13 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typena
     auto sweepHandle = handle.sweep();
 
     for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
+        auto* objHeader = it->GetObjHeader();
         if (Traits::TryResetMark(*it)) {
             ++it;
-            sweepHandle.addKeptObject();
+            sweepHandle.addKeptObject(Traits::ObjectFactory::GetAllocatedHeapSize(objHeader));
             continue;
         }
         sweepHandle.addSweptObject();
-        auto* objHeader = it->GetObjHeader();
         if (HasFinalizers(objHeader)) {
             objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
         } else {

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -5,14 +5,11 @@
 
 #include "ConcurrentMarkAndSweep.hpp"
 
-#include <cinttypes>
 #include <optional>
 
 #include "AllocatorImpl.hpp"
 #include "CallsChecker.hpp"
 #include "CompilerConstants.hpp"
-#include "GlobalData.hpp"
-#include "GCImpl.hpp"
 #include "Logging.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "Memory.h"
@@ -209,7 +206,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     }
     finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
-    scheduler.onGCFinish(epoch, alloc::allocatedBytes());
+    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes());
     state_.finish(epoch);
     gcHandle.finalizersScheduled(finalizerQueue.size());
     gcHandle.finished();

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp

@@ -379,6 +379,21 @@ MarkStats GCHandle::getMarked() {
     return MarkStats{};
 }
 
+size_t GCHandle::getKeptSizeBytes() noexcept {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        size_t result = 0;
+        if (stat->sweepStats.heap) {
+            result += stat->sweepStats.heap->keptSizeBytes;
+        }
+        if (stat->sweepStats.extra) {
+            result += stat->sweepStats.extra->keptSizeBytes;
+        }
+        return result;
+    }
+    return 0;
+}
+
 void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
@@ -388,6 +403,7 @@ void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept {
         }
         heap->keptCount += stats.keptCount;
         heap->sweptCount += stats.sweptCount;
+        heap->keptSizeBytes += stats.keptSizeBytes;
         RuntimeAssert(static_cast<bool>(stat->markStats), "Mark must have already happened");
         stat->markStats->markedCount += markedCount;
     }
@@ -402,6 +418,7 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept {
         }
         extra->keptCount += stats.keptCount;
         extra->sweptCount += stats.sweptCount;
+        extra->keptSizeBytes += stats.keptSizeBytes;
     }
 }
 

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp

@@ -8,7 +8,6 @@
 #include <cstdint>
 #include <optional>
 
-#include "Common.h"
 #include "Logging.hpp"
 #include "Porting.h"
 #include "Utils.hpp"
@@ -28,6 +27,7 @@ class GCHandle;
 struct SweepStats {
     uint64_t sweptCount = 0;
     uint64_t keptCount = 0;
+    size_t keptSizeBytes = 0;
 };
 
 struct MarkStats {
@@ -69,6 +69,7 @@ public:
     [[nodiscard]] GCProcessWeaksScope processWeaks() noexcept;
 
     MarkStats getMarked();
+    size_t getKeptSizeBytes() noexcept;
 
 private:
     uint64_t epoch_;
@@ -117,7 +118,10 @@ public:
     ~GCSweepScope();
 
     void addSweptObject() noexcept { stats_.sweptCount += 1; }
-    void addKeptObject() noexcept { stats_.keptCount += 1; }
+    void addKeptObject(size_t sizeBytes) noexcept {
+        stats_.keptCount += 1;
+        stats_.keptSizeBytes += sizeBytes;
+    }
     // Custom allocator only. To be finalized objects are kept alive.
     void addMarkedObject() noexcept { markedCount_ += 1; }
 };
@@ -132,7 +136,10 @@ public:
     ~GCSweepExtraObjectsScope();
 
     void addSweptObject() noexcept { stats_.sweptCount += 1; }
-    void addKeptObject() noexcept { stats_.keptCount += 1; }
+    void addKeptObject(size_t sizeBytes) noexcept {
+        stats_.keptCount += 1;
+        stats_.keptSizeBytes += sizeBytes;
+    }
 };
 
 class GCHandle::GCGlobalRootSetScope : private GCStageScopeBase {

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -5,17 +5,11 @@
 
 #include "SameThreadMarkAndSweep.hpp"
 
-#include <cinttypes>
-
-#include "CompilerConstants.hpp"
-#include "GlobalData.hpp"
-#include "GCImpl.hpp"
 #include "GCStatistics.hpp"
 #include "Logging.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "Memory.h"
 #include "RootSet.hpp"
-#include "Runtime.h"
 #include "ThreadData.hpp"
 #include "ThreadRegistry.hpp"
 #include "ThreadSuspension.hpp"
@@ -102,7 +96,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
 
-    scheduler.onGCFinish(epoch, alloc::allocatedBytes());
+    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes());
 
     resumeTheWorld(gcHandle);
 

kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp

@@ -11,12 +11,6 @@
 
 namespace kotlin::gcScheduler {
 
-#if KONAN_WATCHOS
-constexpr int64_t kDefaultTargetHeapBytes = 10 * 1024 * 1024;
-#else
-constexpr int64_t kDefaultTargetHeapBytes = 100 * 1024 * 1024;
-#endif
-
 // NOTE: When changing default values, reflect them in GC.kt as well.
 struct GCSchedulerConfig {
     enum class MutatorAssists {
@@ -33,7 +27,7 @@ struct GCSchedulerConfig {
     // become bigger than this value, and `mutatorAssists` are enabled the GC will
     // stop the world and wait until current epoch finishes.
     // Adapts after each GC epoch when `autoTune = true`.
-    std::atomic<int64_t> targetHeapBytes = kDefaultTargetHeapBytes;
+    std::atomic<int64_t> targetHeapBytes = 10 * 1024 * 1024;
     // The rate at which `targetHeapBytes` changes when `autoTune = true`. Concretely: if after the collection
     // `N` object bytes remain in the heap, the next `targetHeapBytes` will be `N / targetHeapUtilization` capped
     // between `minHeapBytes` and `maxHeapBytes`.
@@ -46,7 +40,7 @@ struct GCSchedulerConfig {
     std::atomic<double> heapTriggerCoefficient = 0.9;
     // See `mutatorAssists()`.
     std::atomic<std::underlying_type_t<MutatorAssists>> mutatorAssistsImpl =
-            static_cast<std::underlying_type_t<MutatorAssists>>(MutatorAssists::kDefault);
+            static_cast<std::underlying_type_t<MutatorAssists>>(MutatorAssists::kDisable);
 
     std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); }