[K/N] Track stats in Mark

Tracking alive heap set size after Mark GC phase.

Merge-request: KT-MR-5179
^KT-48537

kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "Alloc.h"
+
+namespace kotlin {
+namespace gc {
+
+class AlignedAllocator {
+public:
+    void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); }
+    static void Free(void* instance) noexcept { konanFreeMemory(instance); }
+};
+
+template <typename BaseAllocator, typename GCThreadData>
+class AllocatorWithGC {
+public:
+    AllocatorWithGC(BaseAllocator base, GCThreadData& gc) noexcept : base_(std::move(base)), gc_(gc) {}
+
+    void* Alloc(size_t size, size_t alignment) noexcept {
+        gc_.SafePointAllocation(size);
+        if (void* ptr = base_.Alloc(size, alignment)) {
+            return ptr;
+        }
+        // Tell GC that we failed to allocate, and try one more time.
+        gc_.OnOOM(size);
+        return base_.Alloc(size, alignment);
+    }
+
+    static void Free(void* instance) noexcept { BaseAllocator::Free(instance); }
+
+private:
+    BaseAllocator base_;
+    GCThreadData& gc_;
+};
+
+} // namespace gc
+} // namespace kotlin

kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "Allocator.hpp"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "Types.h"
+
+using namespace kotlin;
+
+using testing::_;
+
+namespace {
+
+class MockAllocator {
+public:
+    MOCK_METHOD(void*, Alloc, (size_t, size_t));
+};
+
+class MockAllocatorWrapper {
+public:
+    MockAllocator& operator*() { return *mock_; }
+
+    void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); }
+
+private:
+    KStdUniquePtr<testing::StrictMock<MockAllocator>> mock_ = make_unique<testing::StrictMock<MockAllocator>>();
+};
+
+class MockGC {
+public:
+    MOCK_METHOD(void, SafePointAllocation, (size_t));
+    MOCK_METHOD(void, OnOOM, (size_t));
+};
+
+} // namespace
+
+TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    void* nonNull = reinterpret_cast<void*>(1);
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
+        EXPECT_CALL(gc, OnOOM(_)).Times(0);
+    }
+    gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nonNull);
+}
+
+TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    void* nonNull = reinterpret_cast<void*>(1);
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+        EXPECT_CALL(gc, OnOOM(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
+    }
+    gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nonNull);
+}
+
+TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+        EXPECT_CALL(gc, OnOOM(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+    }
+    gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nullptr);
+}

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp

@@ -20,6 +20,7 @@ namespace {
 class GCEmptySchedulerData : public gc::GCSchedulerData {
     void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {}
     void OnPerformFullGC() noexcept override {}
+    void UpdateAliveSetBytes(size_t bytes) noexcept override {}
 };
 
 class GCSchedulerDataWithTimer : public gc::GCSchedulerData {
@@ -40,6 +41,8 @@ public:
 
     void OnPerformFullGC() noexcept override {}
 
+    void UpdateAliveSetBytes(size_t bytes) noexcept override {}
+
 private:
     void OnTimer() noexcept {
         auto allThreadsAreNative = []() {
@@ -83,6 +86,8 @@ public:
 
     void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); }
 
+    void UpdateAliveSetBytes(size_t bytes) noexcept override {}
+
 private:
     gc::GCSchedulerConfig& config_;
     CurrentTimeCallback currentTimeCallbackNs_;

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp

@@ -63,6 +63,9 @@ public:
 
     // Always called by the GC thread.
     virtual void OnPerformFullGC() noexcept = 0;
+
+    // Always called by the GC thread.
+    virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0;
 };
 
 class GCSchedulerThreadData {

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -9,6 +9,7 @@
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooks.hpp"
 #include "Memory.h"
+#include "ObjectOps.hpp"
 #include "ObjectTraversal.hpp"
 #include "Runtime.h"
 #include "Types.h"
@@ -16,9 +17,19 @@
 namespace kotlin {
 namespace gc {
 
+struct MarkStats {
+    // How many objects are alive.
+    size_t aliveHeapSet = 0;
+    // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself.
+    size_t aliveHeapSetBytes = 0;
+    // How many times a marked object was found in the mark queue.
+    size_t duplicateEntries = 0;
+};
+
 // TODO: Because of `graySet` this implementation may allocate heap memory during GC.
 template <typename Traits>
-void Mark(KStdVector<ObjHeader*> graySet) noexcept {
+MarkStats Mark(KStdVector<ObjHeader*> graySet) noexcept {
+    MarkStats stats;
     while (!graySet.empty()) {
         ObjHeader* top = graySet.back();
         graySet.pop_back();
@@ -27,8 +38,11 @@ void Mark(KStdVector<ObjHeader*> graySet) noexcept {
 
         if (top->heap()) {
             if (!Traits::TryMark(top)) {
+                ++stats.duplicateEntries;
                 continue;
             }
+            stats.aliveHeapSet++;
+            stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
         }
 
         if (top->heap() || top->local()) {
@@ -46,6 +60,7 @@ void Mark(KStdVector<ObjHeader*> graySet) noexcept {
             }
         }
     }
+    return stats;
 }
 
 template <typename Traits>

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -143,10 +143,10 @@ public:
         return testing::UnorderedElementsAreArray(objects);
     }
 
-    void Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
+    gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
         KStdVector<ObjHeader*> objects;
         for (auto& object : graySet) objects.push_back(object.get().GetObjHeader());
-        gc::Mark<ScopedMarkTraits>(std::move(objects));
+        return gc::Mark<ScopedMarkTraits>(std::move(objects));
     }
 
 private:
@@ -154,86 +154,118 @@ private:
     ScopedMarkTraits markTraits_;
 };
 
-#define EXPECT_MARKED(...) EXPECT_THAT(marked(), MarkedMatcher({__VA_ARGS__}))
+size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<BaseObject>> objects) {
+    size_t size = 0;
+    for (auto& object : objects) {
+        size += mm::GetAllocatedHeapSize(object.get().GetObjHeader());
+    }
+    return size;
+}
+
+#define EXPECT_MARKED(stats, ...) \
+    do { \
+        std::initializer_list<std::reference_wrapper<BaseObject>> objects = {__VA_ARGS__}; \
+        EXPECT_THAT(stats.aliveHeapSet, objects.size()); \
+        EXPECT_THAT(stats.aliveHeapSetBytes, GetObjectsSize(objects)); \
+        EXPECT_THAT(marked(), MarkedMatcher(objects)); \
+    } while (false)
 
 } // namespace
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkNothing) {
-    Mark({});
+    auto stats = Mark({});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) {
     Object object;
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED(object);
+    EXPECT_MARKED(stats, object);
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) {
     ObjectArray array;
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) {
     CharArray array;
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObject) {
     Object object{BaseObject::Kind::kPermanent};
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObjectArray) {
     ObjectArray array{BaseObject::Kind::kPermanent};
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentCharArray) {
     CharArray array{BaseObject::Kind::kPermanent};
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObject) {
     Object object{BaseObject::Kind::kStackLocal};
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObjectArray) {
     ObjectArray array{BaseObject::Kind::kStackLocal};
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackCharArray) {
     CharArray array{BaseObject::Kind::kStackLocal};
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 
@@ -241,18 +273,22 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) {
     Object object;
     object->field1 = kInitializingSingleton;
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED(object);
+    EXPECT_MARKED(stats, object);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) {
     ObjectArray array;
     array.elements()[0] = kInitializingSingleton;
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) {
@@ -261,36 +297,44 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) {
     array.elements()[1] = 'b';
     array.elements()[2] = 'c';
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) {
     Object object;
     object.InstallExtraData();
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED(object);
+    EXPECT_MARKED(stats, object);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) {
     ObjectArray array;
     array.InstallExtraData();
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) {
     CharArray array;
     array.InstallExtraData();
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array);
+    EXPECT_MARKED(stats, array);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) {
@@ -298,9 +342,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) {
     Object object;
     object.InstallWeakCounter(weakCounter);
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED(object, weakCounter);
+    EXPECT_MARKED(stats, object, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) {
@@ -308,9 +354,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) {
     ObjectArray array;
     array.InstallWeakCounter(weakCounter);
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array, weakCounter);
+    EXPECT_MARKED(stats, array, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) {
@@ -318,9 +366,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) {
     CharArray array;
     array.InstallWeakCounter(weakCounter);
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array, weakCounter);
+    EXPECT_MARKED(stats, array, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) {
@@ -329,9 +379,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCount
     object->field1 = kInitializingSingleton;
     object.InstallWeakCounter(weakCounter);
 
-    Mark({object});
+    auto stats = Mark({object});
 
-    EXPECT_MARKED(object, weakCounter);
+    EXPECT_MARKED(stats, object, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) {
@@ -340,9 +392,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeak
     array.elements()[0] = kInitializingSingleton;
     array.InstallWeakCounter(weakCounter);
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array, weakCounter);
+    EXPECT_MARKED(stats, array, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) {
@@ -353,9 +407,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter
     array.elements()[2] = 'c';
     array.InstallWeakCounter(weakCounter);
 
-    Mark({array});
+    auto stats = Mark({array});
 
-    EXPECT_MARKED(array, weakCounter);
+    EXPECT_MARKED(stats, array, weakCounter);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) {
@@ -375,11 +431,13 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) {
     root_field3.elements()[1] = root_field3_element2.header();
     root_field3.elements()[2] = root_field3_element3.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
     EXPECT_MARKED(
-            root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2,
+            stats, root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2,
             root_field3_element3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) {
@@ -399,9 +457,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) {
     root_field3.elements()[1] = root_field3_element2.header();
     root_field3.elements()[2] = root_field3_element3.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) {
@@ -421,9 +481,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) {
     root_field3.elements()[1] = root_field3_element2.header();
     root_field3.elements()[2] = root_field3_element3.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
-    EXPECT_MARKED(root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
+    EXPECT_MARKED(stats, root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) {
@@ -443,9 +505,12 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) {
     root_field3.elements()[1] = root_field3_element2.header();
     root_field3.elements()[2] = root_field3_element3.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
-    EXPECT_MARKED(root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
+    EXPECT_MARKED(
+            stats, root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) {
@@ -465,9 +530,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) {
     root_field3.elements()[1] = root_field3_element2.header();
     root_field3.elements()[2] = root_field3_element3.header();
 
-    Mark({root, root_field3});
+    auto stats = Mark({root, root_field3});
 
-    EXPECT_MARKED(root_field3_element1, root_field3_element2);
+    EXPECT_MARKED(stats, root_field3_element1, root_field3_element2);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) {
@@ -478,9 +545,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) {
     inner1->field1 = inner2.header();
     inner2.elements()[0] = root.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
-    EXPECT_MARKED(root, inner1, inner2);
+    EXPECT_MARKED(stats, root, inner1, inner2);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) {
@@ -491,9 +560,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) {
     inner1->field1 = inner2.header();
     inner2.elements()[0] = root.header();
 
-    Mark({root});
+    auto stats = Mark({root});
 
-    EXPECT_MARKED();
+    EXPECT_MARKED(stats);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) {
@@ -509,9 +580,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) {
     root->field2 = inner3.header();
     inner3->field1 = inner2.header();
 
-    Mark({root, inner1, inner2});
+    auto stats = Mark({root, inner1, inner2});
 
-    EXPECT_MARKED(inner3, inner2_element1);
+    EXPECT_MARKED(stats, inner3, inner2_element1);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 
@@ -520,9 +593,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) {
     ObjectArray root2;
     Object root3;
 
-    Mark({root1, root2, root3});
+    auto stats = Mark({root1, root2, root3});
 
-    EXPECT_MARKED(root1, root2, root3);
+    EXPECT_MARKED(stats, root1, root2, root3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) {
@@ -530,9 +605,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) {
     ObjectArray root2;
     Object root3;
 
-    Mark({root1, root2, root3});
+    auto stats = Mark({root1, root2, root3});
 
-    EXPECT_MARKED(root2, root3);
+    EXPECT_MARKED(stats, root2, root3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) {
@@ -540,9 +617,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) {
     ObjectArray root2{BaseObject::Kind::kPermanent};
     Object root3;
 
-    Mark({root1, root2, root3});
+    auto stats = Mark({root1, root2, root3});
 
-    EXPECT_MARKED(root1, root3);
+    EXPECT_MARKED(stats, root1, root3);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) {
@@ -550,9 +629,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) {
     ObjectArray root2;
     Object root3{BaseObject::Kind::kPermanent};
 
-    Mark({root1, root2, root3});
+    auto stats = Mark({root1, root2, root3});
 
-    EXPECT_MARKED(root1, root2);
+    EXPECT_MARKED(stats, root1, root2);
+
+    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) {
@@ -564,7 +645,9 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) {
     root2.elements()[0] = root3.header();
     root3->field1 = root1.header();
 
-    Mark({root1, root2, root3});
+    auto stats = Mark({root1, root2, root3});
 
-    EXPECT_MARKED(root1, root2, root3);
+    EXPECT_MARKED(stats, root1, root2, root3);
+
+    EXPECT_THAT(stats.duplicateEntries, 2);
 }

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -53,9 +53,12 @@ struct GC {
         State state = State::kUnmarked;
     };
 
+    using Allocator = gc::AlignedAllocator;
+
     struct ThreadData {
         void SafePointAllocation(size_t) {}
         void OnOOM(size_t) {}
+        Allocator CreateAllocator() { return Allocator(); }
     };
 };
 

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp

@@ -8,6 +8,7 @@
 
 #include <cstddef>
 
+#include "Allocator.hpp"
 #include "GCScheduler.hpp"
 #include "Utils.hpp"
 #include "Types.h"
@@ -26,6 +27,8 @@ class NoOpGC : private Pinned {
 public:
     class ObjectData {};
 
+    using Allocator = AlignedAllocator;
+
     class ThreadData : private Pinned {
     public:
         using ObjectData = NoOpGC::ObjectData;
@@ -41,6 +44,8 @@ public:
 
         void OnOOM(size_t size) noexcept {}
 
+        Allocator CreateAllocator() noexcept { return Allocator(); }
+
     private:
     };
 

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -193,9 +193,10 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         RuntimeLogInfo(
                 {kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(),
                 stableRef, timeRootSetUs - timeSuspendUs);
-        gc::Mark<MarkTraits>(std::move(graySet));
+        auto markStats = gc::Mark<MarkTraits>(std::move(graySet));
         auto timeMarkUs = konan::getTimeMicros();
-        RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs);
+        RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries);
+        scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
         gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
         auto timeSweepExtraObjectsUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -8,6 +8,7 @@
 
 #include <cstddef>
 
+#include "Allocator.hpp"
 #include "GCScheduler.hpp"
 #include "ObjectFactory.hpp"
 #include "Types.h"
@@ -48,6 +49,7 @@ public:
     class ThreadData : private Pinned {
     public:
         using ObjectData = SameThreadMarkAndSweep::ObjectData;
+        using Allocator = AllocatorWithGC<AlignedAllocator, ThreadData>;
 
         explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
         ~ThreadData() = default;
@@ -61,6 +63,8 @@ public:
 
         void OnOOM(size_t size) noexcept;
 
+        Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); }
+
     private:
         void SafePointRegular(size_t weight) noexcept;
         void SafePointSlowPath(SafepointFlag flag) noexcept;
@@ -69,6 +73,8 @@ public:
         mm::ThreadData& threadData_;
     };
 
+    using Allocator = ThreadData::Allocator;
+
     SameThreadMarkAndSweep() noexcept;
     ~SameThreadMarkAndSweep() = default;
 

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -53,6 +53,13 @@ public:
     public:
         ~Node() = default;
 
+        constexpr static std::pair<size_t, size_t> GetSizeAndAlignmentForDataSize(size_t dataSize) noexcept {
+            size_t dataSizeAligned = AlignUp(dataSize, DataAlignment);
+            size_t totalAlignment = std::max(alignof(Node), DataAlignment);
+            size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment);
+            return std::make_pair(totalSize, totalAlignment);
+        }
+
         static Node& FromData(void* data) noexcept {
             constexpr size_t kDataOffset = DataOffset();
             Node* node = reinterpret_cast<Node*>(reinterpret_cast<uintptr_t>(data) - kDataOffset);
@@ -80,9 +87,7 @@ public:
         Node() noexcept = default;
 
         static unique_ptr<Node> Create(Allocator& allocator, size_t dataSize) noexcept {
-            size_t dataSizeAligned = AlignUp(dataSize, DataAlignment);
+            auto [totalSize, totalAlignment] = GetSizeAndAlignmentForDataSize(dataSize);
-            size_t totalAlignment = std::max(alignof(Node), DataAlignment);
-            size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment);
             RuntimeAssert(
                     DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize,
                     DataOffset());
@@ -392,43 +397,13 @@ private:
     SpinLock<MutexThreadStateHandling::kIgnore> mutex_;
 };
 
-class SimpleAllocator {
-public:
-    void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); }
-
-    static void Free(void* instance) noexcept { konanFreeMemory(instance); }
-};
-
-template <typename BaseAllocator, typename GC>
-class AllocatorWithGC {
-public:
-    AllocatorWithGC(BaseAllocator base, GC& gc) noexcept : base_(std::move(base)), gc_(gc) {}
-
-    void* Alloc(size_t size, size_t alignment) noexcept {
-        gc_.SafePointAllocation(size);
-        if (void* ptr = base_.Alloc(size, alignment)) {
-            return ptr;
-        }
-        // Tell GC that we failed to allocate, and try one more time.
-        gc_.OnOOM(size);
-        return base_.Alloc(size, alignment);
-    }
-
-    static void Free(void* instance) noexcept { BaseAllocator::Free(instance); }
-
-private:
-    BaseAllocator base_;
-    GC& gc_;
-};
-
 } // namespace internal
 
 template <typename GC>
 class ObjectFactory : private Pinned {
     using GCObjectData = typename GC::ObjectData;
     using GCThreadData = typename GC::ThreadData;
-
+    using Allocator = typename GC::Allocator;
-    using Allocator = internal::AllocatorWithGC<internal::SimpleAllocator, GCThreadData>;
 
     struct HeapObjHeader {
         GCObjectData gcData;
@@ -523,13 +498,17 @@ public:
             typename Storage::Producer::Iterator iterator_;
         };
 
-        ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept :
+        ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept : producer_(owner.storage_, gc.CreateAllocator()) {}
-            producer_(owner.storage_, internal::AllocatorWithGC(internal::SimpleAllocator(), gc)) {}
+
+        static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept {
+            RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
+            size_t allocSize = ObjectAllocatedDataSize(typeInfo);
+            return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first;
+        }
 
         ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept {
             RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
-            size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
+            size_t allocSize = ObjectAllocatedDataSize(typeInfo);
-            size_t allocSize = AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
             auto& node = producer_.Insert(allocSize);
             auto* heapObject = new (node.Data()) HeapObjHeader();
             auto* object = &heapObject->object;
@@ -538,11 +517,15 @@ public:
             return object;
         }
 
+        static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
+            RuntimeAssert(typeInfo->IsArray(), "Must be an array");
+            size_t allocSize = ArrayAllocatedDataSize(typeInfo, count);
+            return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first;
+        }
+
         ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
             RuntimeAssert(typeInfo->IsArray(), "Must be an array");
-            uint32_t membersSize = static_cast<uint32_t>(-typeInfo->instanceSize_) * count;
+            size_t allocSize = ArrayAllocatedDataSize(typeInfo, count);
-            // Note: array body is aligned, but for size computation it is enough to align the sum.
-            size_t allocSize = AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
             auto& node = producer_.Insert(allocSize);
             auto* heapArray = new (node.Data()) HeapArrayHeader();
             auto* array = &heapArray->array;
@@ -560,6 +543,17 @@ public:
         void ClearForTests() noexcept { producer_.ClearForTests(); }
 
     private:
+        static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
+            size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
+            return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
+        }
+
+        static size_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
+            uint32_t membersSize = static_cast<uint32_t>(-typeInfo->instanceSize_) * count;
+            // Note: array body is aligned, but for size computation it is enough to align the sum.
+            return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
+        }
+
         typename Storage::Producer producer_;
     };
 

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -12,7 +12,9 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "Allocator.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "ObjectOps.hpp"
 #include "ObjectTestSupport.hpp"
 #include "TestSupport.hpp"
 #include "Types.h"
@@ -23,7 +25,7 @@ using testing::_;
 
 namespace {
 
-using SimpleAllocator = mm::internal::SimpleAllocator;
+using SimpleAllocator = gc::AlignedAllocator;
 
 template <size_t DataAlignment>
 using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator>;
@@ -691,86 +693,61 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
     EXPECT_THAT(storage.GetSizeUnsafe(), expectedAfter.size());
 }
 
-using mm::internal::AllocatorWithGC;
-
 namespace {
 
-class MockAllocator {
+class MockAllocator : private Pinned {
 public:
+    MockAllocator();
+    ~MockAllocator();
+
     MOCK_METHOD(void*, Alloc, (size_t, size_t));
+    MOCK_METHOD(void, Free, (void*));
+
+    void* DefaultAlloc(size_t size, size_t alignment) { return konanAllocAlignedMemory(size, alignment); }
+
+    void DefaultFree(void* instance) { konanFreeMemory(instance); }
 };
 
-class MockAllocatorWrapper {
+class GlobalMockAllocator {
 public:
-    MockAllocator& operator*() { return *mock_; }
+    void* Alloc(size_t size, size_t alignment) {
+        RuntimeAssert(instance_ != nullptr, "Global allocator must be set");
+        return instance_->Alloc(size, alignment);
+    }
 
-    void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); }
+    static void Free(void* instance) {
+        RuntimeAssert(instance_ != nullptr, "Global allocator must be set");
+        instance_->Free(instance);
+    }
+
+    static void SetMockAllocator(MockAllocator* instance) {
+        RuntimeAssert(instance != nullptr, "Cannot be null");
+        RuntimeAssert(instance_ == nullptr, "No global allocator can be set");
+        instance_ = instance;
+    }
+
+    static void ClearMockAllocator(MockAllocator* instance) {
+        RuntimeAssert(instance != nullptr, "Cannot be null");
+        RuntimeAssert(instance_ == instance, "Allocators must match");
+        instance_ = nullptr;
+    }
 
 private:
-    KStdUniquePtr<testing::StrictMock<MockAllocator>> mock_ = make_unique<testing::StrictMock<MockAllocator>>();
+    static MockAllocator* instance_;
 };
 
-class MockGC {
+MockAllocator::MockAllocator() {
-public:
+    GlobalMockAllocator::SetMockAllocator(this);
-    MOCK_METHOD(void, SafePointAllocation, (size_t));
+    ON_CALL(*this, Alloc(_, _)).WillByDefault([this](size_t size, size_t alignment) { return DefaultAlloc(size, alignment); });
-    MOCK_METHOD(void, OnOOM, (size_t));
+    ON_CALL(*this, Free(_)).WillByDefault([this](void* instance) { DefaultFree(instance); });
-};
-
-} // namespace
-
-TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
-    constexpr size_t size = 256;
-    constexpr size_t alignment = 8;
-    void* nonNull = reinterpret_cast<void*>(1);
-    MockAllocatorWrapper baseAllocator;
-    testing::StrictMock<MockGC> gc;
-    {
-        testing::InSequence seq;
-        EXPECT_CALL(gc, SafePointAllocation(size));
-        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
-        EXPECT_CALL(gc, OnOOM(_)).Times(0);
-    }
-    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
-    void* ptr = allocator.Alloc(size, alignment);
-    EXPECT_THAT(ptr, nonNull);
 }
 
-TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
+MockAllocator::~MockAllocator() {
-    constexpr size_t size = 256;
+    GlobalMockAllocator::ClearMockAllocator(this);
-    constexpr size_t alignment = 8;
-    void* nonNull = reinterpret_cast<void*>(1);
-    MockAllocatorWrapper baseAllocator;
-    testing::StrictMock<MockGC> gc;
-    {
-        testing::InSequence seq;
-        EXPECT_CALL(gc, SafePointAllocation(size));
-        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
-        EXPECT_CALL(gc, OnOOM(size));
-        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
-    }
-    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
-    void* ptr = allocator.Alloc(size, alignment);
-    EXPECT_THAT(ptr, nonNull);
 }
 
-TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
+// static
-    constexpr size_t size = 256;
+MockAllocator* GlobalMockAllocator::instance_ = nullptr;
-    constexpr size_t alignment = 8;
-    MockAllocatorWrapper baseAllocator;
-    testing::StrictMock<MockGC> gc;
-    {
-        testing::InSequence seq;
-        EXPECT_CALL(gc, SafePointAllocation(size));
-        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
-        EXPECT_CALL(gc, OnOOM(size));
-        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
-    }
-    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
-    void* ptr = allocator.Alloc(size, alignment);
-    EXPECT_THAT(ptr, nullptr);
-}
-
-namespace {
 
 class GC {
 public:
@@ -778,11 +755,15 @@ public:
         uint32_t flags = 42;
     };
 
+    using Allocator = GlobalMockAllocator;
+
     class ThreadData {
     public:
         void SafePointAllocation(size_t size) noexcept {}
 
         void OnOOM(size_t size) noexcept {}
+
+        Allocator CreateAllocator() noexcept { return Allocator(); }
     };
 };
 
@@ -801,12 +782,26 @@ struct Payload {
 } // namespace
 
 TEST(ObjectFactoryTest, CreateObject) {
+    testing::StrictMock<MockAllocator> allocator;
+
     test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
     GC::ThreadData gc;
     ObjectFactory objectFactory;
     ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
+    size_t allocSize = 0;
+    void* allocAddress = nullptr;
+    EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) {
+        allocSize = size;
+        allocAddress = allocator.DefaultAlloc(size, alignment);
+        return allocAddress;
+    });
     auto* object = threadQueue.CreateObject(type.typeInfo());
+    testing::Mock::VerifyAndClearExpectations(&allocator);
+    EXPECT_THAT(allocSize, testing::Gt<size_t>(type.typeInfo()->instanceSize_));
+    EXPECT_THAT(allocAddress, testing::Ne(nullptr));
+    EXPECT_THAT(mm::GetAllocatedHeapSize(object), allocSize);
+
     threadQueue.Publish();
 
     auto node = ObjectFactory::NodeRef::From(object);
@@ -819,14 +814,30 @@ TEST(ObjectFactoryTest, CreateObject) {
     EXPECT_THAT(*it, node);
     ++it;
     EXPECT_THAT(it, iter.end());
+
+    EXPECT_CALL(allocator, Free(allocAddress));
 }
 
 TEST(ObjectFactoryTest, CreateObjectArray) {
+    testing::StrictMock<MockAllocator> allocator;
+
     GC::ThreadData gc;
     ObjectFactory objectFactory;
     ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
+    size_t allocSize = 0;
+    void* allocAddress = nullptr;
+    EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) {
+        allocSize = size;
+        allocAddress = allocator.DefaultAlloc(size, alignment);
+        return allocAddress;
+    });
     auto* array = threadQueue.CreateArray(theArrayTypeInfo, 3);
+    testing::Mock::VerifyAndClearExpectations(&allocator);
+    EXPECT_THAT(allocSize, testing::Gt<size_t>(-theArrayTypeInfo->instanceSize_ * 3));
+    EXPECT_THAT(allocAddress, testing::Ne(nullptr));
+    EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize);
+
     threadQueue.Publish();
 
     auto node = ObjectFactory::NodeRef::From(array);
@@ -839,14 +850,30 @@ TEST(ObjectFactoryTest, CreateObjectArray) {
     EXPECT_THAT(*it, node);
     ++it;
     EXPECT_THAT(it, iter.end());
+
+    EXPECT_CALL(allocator, Free(allocAddress));
 }
 
 TEST(ObjectFactoryTest, CreateCharArray) {
+    testing::StrictMock<MockAllocator> allocator;
+
     GC::ThreadData gc;
     ObjectFactory objectFactory;
     ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
+    size_t allocSize = 0;
+    void* allocAddress = nullptr;
+    EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) {
+        allocSize = size;
+        allocAddress = allocator.DefaultAlloc(size, alignment);
+        return allocAddress;
+    });
     auto* array = threadQueue.CreateArray(theCharArrayTypeInfo, 3);
+    testing::Mock::VerifyAndClearExpectations(&allocator);
+    EXPECT_THAT(allocSize, testing::Gt<size_t>(-theCharArrayTypeInfo->instanceSize_ * 3));
+    EXPECT_THAT(allocAddress, testing::Ne(nullptr));
+    EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize);
+
     threadQueue.Publish();
 
     auto node = ObjectFactory::NodeRef::From(array);
@@ -859,18 +886,24 @@ TEST(ObjectFactoryTest, CreateCharArray) {
     EXPECT_THAT(*it, node);
     ++it;
     EXPECT_THAT(it, iter.end());
+
+    EXPECT_CALL(allocator, Free(allocAddress));
 }
 
 TEST(ObjectFactoryTest, Erase) {
+    testing::StrictMock<MockAllocator> allocator;
+
     test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
     GC::ThreadData gc;
     ObjectFactory objectFactory;
     ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
+    EXPECT_CALL(allocator, Alloc(_, _)).Times(20);
     for (int i = 0; i < 10; ++i) {
         threadQueue.CreateObject(objectType.typeInfo());
         threadQueue.CreateArray(theArrayTypeInfo, 3);
     }
+    testing::Mock::VerifyAndClearExpectations(&allocator);
 
     threadQueue.Publish();
 
@@ -878,7 +911,9 @@ TEST(ObjectFactoryTest, Erase) {
         auto iter = objectFactory.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->IsArray()) {
+                EXPECT_CALL(allocator, Free(_));
                 iter.EraseAndAdvance(it);
+                testing::Mock::VerifyAndClearExpectations(&allocator);
             } else {
                 ++it;
             }
@@ -893,19 +928,24 @@ TEST(ObjectFactoryTest, Erase) {
         }
         EXPECT_THAT(count, 10);
     }
+    EXPECT_CALL(allocator, Free(_)).Times(10);
 }
 
 TEST(ObjectFactoryTest, Move) {
+    testing::StrictMock<MockAllocator> allocator;
+
     test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
     GC::ThreadData gc;
     ObjectFactory objectFactory;
     ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
     ObjectFactory::FinalizerQueue finalizerQueue;
 
+    EXPECT_CALL(allocator, Alloc(_, _)).Times(20);
     for (int i = 0; i < 10; ++i) {
         threadQueue.CreateObject(objectType.typeInfo());
         threadQueue.CreateArray(theArrayTypeInfo, 3);
     }
+    testing::Mock::VerifyAndClearExpectations(&allocator);
 
     threadQueue.Publish();
 
@@ -937,9 +977,13 @@ TEST(ObjectFactoryTest, Move) {
         }
         EXPECT_THAT(count, 10);
     }
+
+    EXPECT_CALL(allocator, Free(_)).Times(20);
 }
 
 TEST(ObjectFactoryTest, RunFinalizers) {
+    testing::StrictMock<MockAllocator> allocator;
+
     FinalizerHooksTestSupport finalizerHooks;
 
     test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder<Payload>().addFlag(TF_HAS_FINALIZER)};
@@ -949,9 +993,11 @@ TEST(ObjectFactoryTest, RunFinalizers) {
     ObjectFactory::FinalizerQueue finalizerQueue;
 
     KStdVector<ObjHeader*> objects;
+    EXPECT_CALL(allocator, Alloc(_, _)).Times(10);
     for (int i = 0; i < 10; ++i) {
         objects.push_back(threadQueue.CreateObject(objectType.typeInfo()));
     }
+    testing::Mock::VerifyAndClearExpectations(&allocator);
 
     threadQueue.Publish();
 
@@ -968,9 +1014,12 @@ TEST(ObjectFactoryTest, RunFinalizers) {
     finalizerQueue.Finalize();
     // Hooks called before `FinalizerQueue` destructor.
     testing::Mock::VerifyAndClearExpectations(&finalizerHooks.finalizerHook());
+    EXPECT_CALL(allocator, Free(_)).Times(10);
 }
 
 TEST(ObjectFactoryTest, ConcurrentPublish) {
+    testing::StrictMock<MockAllocator> allocator;
+
     test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
     ObjectFactory objectFactory;
     constexpr int kThreadCount = kDefaultThreadCount;
@@ -980,6 +1029,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
     std::mutex expectedMutex;
     KStdVector<ObjHeader*> expected;
 
+    EXPECT_CALL(allocator, Alloc(_, _)).Times(kThreadCount);
     for (int i = 0; i < kThreadCount; ++i) {
         threads.emplace_back([&type, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() {
             GC::ThreadData gc;
@@ -998,6 +1048,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
 
     while (readyCount < kThreadCount) {
     }
+    testing::Mock::VerifyAndClearExpectations(&allocator);
     canStart = true;
     for (auto& t : threads) {
         t.join();
@@ -1010,4 +1061,5 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
     }
 
     EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
+    EXPECT_CALL(allocator, Free(_)).Times(kThreadCount);
 }

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -69,3 +69,13 @@ OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo,
     // `ArrayHeader` and `ObjHeader` are expected to be compatible.
     RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
 }
+
+size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept {
+    RuntimeAssert(object->heap(), "Object must be a heap object");
+    const auto* typeInfo = object->type_info();
+    if (typeInfo->IsArray()) {
+        return mm::ObjectFactory<gc::GC>::ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_);
+    } else {
+        return mm::ObjectFactory<gc::GC>::ThreadQueue::ObjectAllocatedSize(typeInfo);
+    }
+}

kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp

@@ -29,6 +29,9 @@ OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, Obj
 OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept;
 OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;
 
+// This does not take into account how much storage did the underlying allocator (malloc/mimalloc) reserved.
+size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
+
 } // namespace mm
 } // namespace kotlin