From c57df1ac40f2ad6a8fc81c3b160ef1db9ab2e2ac Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Mon, 6 Dec 2021 14:51:01 +0000 Subject: [PATCH] [K/N] Track stats in Mark Tracking alive heap set size after Mark GC phase. Merge-request: KT-MR-5179 ^KT-48537 --- .../runtime/src/gc/common/cpp/Allocator.hpp | 42 ++++ .../src/gc/common/cpp/AllocatorTest.cpp | 92 +++++++ .../runtime/src/gc/common/cpp/GCScheduler.cpp | 5 + .../runtime/src/gc/common/cpp/GCScheduler.hpp | 3 + .../src/gc/common/cpp/MarkAndSweepUtils.hpp | 17 +- .../common/cpp/MarkAndSweepUtilsMarkTest.cpp | 229 ++++++++++++------ .../common/cpp/MarkAndSweepUtilsSweepTest.cpp | 3 + .../runtime/src/gc/noop/cpp/NoOpGC.hpp | 5 + .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 5 +- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 6 + .../runtime/src/mm/cpp/ObjectFactory.hpp | 76 +++--- .../runtime/src/mm/cpp/ObjectFactoryTest.cpp | 182 +++++++++----- .../runtime/src/mm/cpp/ObjectOps.cpp | 10 + .../runtime/src/mm/cpp/ObjectOps.hpp | 3 + 14 files changed, 496 insertions(+), 182 deletions(-) create mode 100644 kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp create mode 100644 kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp diff --git a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp new file mode 100644 index 00000000000..6d2617a6266 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Alloc.h" + +namespace kotlin { +namespace gc { + +class AlignedAllocator { +public: + void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); } + static void Free(void* instance) noexcept { konanFreeMemory(instance); } +}; + +template +class AllocatorWithGC { +public: + AllocatorWithGC(BaseAllocator base, GCThreadData& gc) noexcept : base_(std::move(base)), gc_(gc) {} + + void* Alloc(size_t size, size_t alignment) noexcept { + gc_.SafePointAllocation(size); + if (void* ptr = base_.Alloc(size, alignment)) { + return ptr; + } + // Tell GC that we failed to allocate, and try one more time. + gc_.OnOOM(size); + return base_.Alloc(size, alignment); + } + + static void Free(void* instance) noexcept { BaseAllocator::Free(instance); } + +private: + BaseAllocator base_; + GCThreadData& gc_; +}; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp new file mode 100644 index 00000000000..6bd3a9f7d3d --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Allocator.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "Types.h" + +using namespace kotlin; + +using testing::_; + +namespace { + +class MockAllocator { +public: + MOCK_METHOD(void*, Alloc, (size_t, size_t)); +}; + +class MockAllocatorWrapper { +public: + MockAllocator& operator*() { return *mock_; } + + void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); } + +private: + KStdUniquePtr> mock_ = make_unique>(); +}; + +class MockGC { +public: + MOCK_METHOD(void, SafePointAllocation, (size_t)); + MOCK_METHOD(void, OnOOM, (size_t)); +}; + +} // namespace + +TEST(AllocatorWithGCTest, AllocateWithoutOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + void* nonNull = reinterpret_cast(1); + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + EXPECT_CALL(gc, OnOOM(_)).Times(0); + } + gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nonNull); +} + +TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + void* nonNull = reinterpret_cast(1); + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(gc, OnOOM(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + } + gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nonNull); +} + +TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(gc, OnOOM(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + } + gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nullptr); +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp index 2213e00c0ee..93700a13f46 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp @@ -20,6 +20,7 @@ namespace { class GCEmptySchedulerData : public gc::GCSchedulerData { void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {} void OnPerformFullGC() noexcept override {} + void UpdateAliveSetBytes(size_t bytes) noexcept override {} }; class GCSchedulerDataWithTimer : public gc::GCSchedulerData { @@ -40,6 +41,8 @@ public: void OnPerformFullGC() noexcept override {} + void UpdateAliveSetBytes(size_t bytes) noexcept override {} + private: void OnTimer() noexcept { auto allThreadsAreNative = []() { @@ -83,6 +86,8 @@ public: void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); } + void UpdateAliveSetBytes(size_t bytes) noexcept override {} + private: gc::GCSchedulerConfig& config_; CurrentTimeCallback currentTimeCallbackNs_; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp index 3e1a8dc6999..28632611a06 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp @@ -63,6 +63,9 @@ public: // Always called by the GC thread. virtual void OnPerformFullGC() noexcept = 0; + + // Always called by the GC thread. + virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0; }; class GCSchedulerThreadData { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 909892d22b9..067a87b3c25 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -9,6 +9,7 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooks.hpp" #include "Memory.h" +#include "ObjectOps.hpp" #include "ObjectTraversal.hpp" #include "Runtime.h" #include "Types.h" @@ -16,9 +17,19 @@ namespace kotlin { namespace gc { +struct MarkStats { + // How many objects are alive. + size_t aliveHeapSet = 0; + // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself. + size_t aliveHeapSetBytes = 0; + // How many times a marked object was found in the mark queue. + size_t duplicateEntries = 0; +}; + // TODO: Because of `graySet` this implementation may allocate heap memory during GC. template -void Mark(KStdVector graySet) noexcept { +MarkStats Mark(KStdVector graySet) noexcept { + MarkStats stats; while (!graySet.empty()) { ObjHeader* top = graySet.back(); graySet.pop_back(); @@ -27,8 +38,11 @@ void Mark(KStdVector graySet) noexcept { if (top->heap()) { if (!Traits::TryMark(top)) { + ++stats.duplicateEntries; continue; } + stats.aliveHeapSet++; + stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top); } if (top->heap() || top->local()) { @@ -46,6 +60,7 @@ void Mark(KStdVector graySet) noexcept { } } } + return stats; } template diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index c07a07f8c52..a7178d2eda9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -143,10 +143,10 @@ public: return testing::UnorderedElementsAreArray(objects); } - void Mark(std::initializer_list> graySet) { + gc::MarkStats Mark(std::initializer_list> graySet) { KStdVector objects; for (auto& object : graySet) objects.push_back(object.get().GetObjHeader()); - gc::Mark(std::move(objects)); + return gc::Mark(std::move(objects)); } private: @@ -154,86 +154,118 @@ private: ScopedMarkTraits markTraits_; }; -#define EXPECT_MARKED(...) EXPECT_THAT(marked(), MarkedMatcher({__VA_ARGS__})) +size_t GetObjectsSize(std::initializer_list> objects) { + size_t size = 0; + for (auto& object : objects) { + size += mm::GetAllocatedHeapSize(object.get().GetObjHeader()); + } + return size; +} + +#define EXPECT_MARKED(stats, ...) \ + do { \ + std::initializer_list> objects = {__VA_ARGS__}; \ + EXPECT_THAT(stats.aliveHeapSet, objects.size()); \ + EXPECT_THAT(stats.aliveHeapSetBytes, GetObjectsSize(objects)); \ + EXPECT_THAT(marked(), MarkedMatcher(objects)); \ + } while (false) } // namespace TEST_F(MarkAndSweepUtilsMarkTest, MarkNothing) { - Mark({}); + auto stats = Mark({}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) { Object object; - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(object); + EXPECT_MARKED(stats, object); + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) { ObjectArray array; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) { CharArray array; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObject) { Object object{BaseObject::Kind::kPermanent}; - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObjectArray) { ObjectArray array{BaseObject::Kind::kPermanent}; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentCharArray) { CharArray array{BaseObject::Kind::kPermanent}; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObject) { Object object{BaseObject::Kind::kStackLocal}; - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObjectArray) { ObjectArray array{BaseObject::Kind::kStackLocal}; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackCharArray) { CharArray array{BaseObject::Kind::kStackLocal}; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } @@ -241,18 +273,22 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) { Object object; object->field1 = kInitializingSingleton; - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(object); + EXPECT_MARKED(stats, object); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) { ObjectArray array; array.elements()[0] = kInitializingSingleton; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) { @@ -261,36 +297,44 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) { array.elements()[1] = 'b'; array.elements()[2] = 'c'; - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) { Object object; object.InstallExtraData(); - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(object); + EXPECT_MARKED(stats, object); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) { ObjectArray array; array.InstallExtraData(); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) { CharArray array; array.InstallExtraData(); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array); + EXPECT_MARKED(stats, array); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) { @@ -298,9 +342,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) { Object object; object.InstallWeakCounter(weakCounter); - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(object, weakCounter); + EXPECT_MARKED(stats, object, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) { @@ -308,9 +354,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) { ObjectArray array; array.InstallWeakCounter(weakCounter); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array, weakCounter); + EXPECT_MARKED(stats, array, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) { @@ -318,9 +366,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) { CharArray array; array.InstallWeakCounter(weakCounter); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array, weakCounter); + EXPECT_MARKED(stats, array, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) { @@ -329,9 +379,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCount object->field1 = kInitializingSingleton; object.InstallWeakCounter(weakCounter); - Mark({object}); + auto stats = Mark({object}); - EXPECT_MARKED(object, weakCounter); + EXPECT_MARKED(stats, object, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) { @@ -340,9 +392,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeak array.elements()[0] = kInitializingSingleton; array.InstallWeakCounter(weakCounter); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array, weakCounter); + EXPECT_MARKED(stats, array, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) { @@ -353,9 +407,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter array.elements()[2] = 'c'; array.InstallWeakCounter(weakCounter); - Mark({array}); + auto stats = Mark({array}); - EXPECT_MARKED(array, weakCounter); + EXPECT_MARKED(stats, array, weakCounter); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) { @@ -375,11 +431,13 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) { root_field3.elements()[1] = root_field3_element2.header(); root_field3.elements()[2] = root_field3_element3.header(); - Mark({root}); + auto stats = Mark({root}); EXPECT_MARKED( - root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, + stats, root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) { @@ -399,9 +457,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) { root_field3.elements()[1] = root_field3_element2.header(); root_field3.elements()[2] = root_field3_element3.header(); - Mark({root}); + auto stats = Mark({root}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) { @@ -421,9 +481,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) { root_field3.elements()[1] = root_field3_element2.header(); root_field3.elements()[2] = root_field3_element3.header(); - Mark({root}); + auto stats = Mark({root}); - EXPECT_MARKED(root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); + EXPECT_MARKED(stats, root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) { @@ -443,9 +505,12 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) { root_field3.elements()[1] = root_field3_element2.header(); root_field3.elements()[2] = root_field3_element3.header(); - Mark({root}); + auto stats = Mark({root}); - EXPECT_MARKED(root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); + EXPECT_MARKED( + stats, root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) { @@ -465,9 +530,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) { root_field3.elements()[1] = root_field3_element2.header(); root_field3.elements()[2] = root_field3_element3.header(); - Mark({root, root_field3}); + auto stats = Mark({root, root_field3}); - EXPECT_MARKED(root_field3_element1, root_field3_element2); + EXPECT_MARKED(stats, root_field3_element1, root_field3_element2); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) { @@ -478,9 +545,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) { inner1->field1 = inner2.header(); inner2.elements()[0] = root.header(); - Mark({root}); + auto stats = Mark({root}); - EXPECT_MARKED(root, inner1, inner2); + EXPECT_MARKED(stats, root, inner1, inner2); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) { @@ -491,9 +560,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) { inner1->field1 = inner2.header(); inner2.elements()[0] = root.header(); - Mark({root}); + auto stats = Mark({root}); - EXPECT_MARKED(); + EXPECT_MARKED(stats); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) { @@ -509,9 +580,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) { root->field2 = inner3.header(); inner3->field1 = inner2.header(); - Mark({root, inner1, inner2}); + auto stats = Mark({root, inner1, inner2}); - EXPECT_MARKED(inner3, inner2_element1); + EXPECT_MARKED(stats, inner3, inner2_element1); + + EXPECT_THAT(stats.duplicateEntries, 0); } @@ -520,9 +593,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) { ObjectArray root2; Object root3; - Mark({root1, root2, root3}); + auto stats = Mark({root1, root2, root3}); - EXPECT_MARKED(root1, root2, root3); + EXPECT_MARKED(stats, root1, root2, root3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) { @@ -530,9 +605,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) { ObjectArray root2; Object root3; - Mark({root1, root2, root3}); + auto stats = Mark({root1, root2, root3}); - EXPECT_MARKED(root2, root3); + EXPECT_MARKED(stats, root2, root3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) { @@ -540,9 +617,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) { ObjectArray root2{BaseObject::Kind::kPermanent}; Object root3; - Mark({root1, root2, root3}); + auto stats = Mark({root1, root2, root3}); - EXPECT_MARKED(root1, root3); + EXPECT_MARKED(stats, root1, root3); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) { @@ -550,9 +629,11 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) { ObjectArray root2; Object root3{BaseObject::Kind::kPermanent}; - Mark({root1, root2, root3}); + auto stats = Mark({root1, root2, root3}); - EXPECT_MARKED(root1, root2); + EXPECT_MARKED(stats, root1, root2); + + EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) { @@ -564,7 +645,9 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) { root2.elements()[0] = root3.header(); root3->field1 = root1.header(); - Mark({root1, root2, root3}); + auto stats = Mark({root1, root2, root3}); - EXPECT_MARKED(root1, root2, root3); + EXPECT_MARKED(stats, root1, root2, root3); + + EXPECT_THAT(stats.duplicateEntries, 2); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index fc0df507893..83fb550b107 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -53,9 +53,12 @@ struct GC { State state = State::kUnmarked; }; + using Allocator = gc::AlignedAllocator; + struct ThreadData { void SafePointAllocation(size_t) {} void OnOOM(size_t) {} + Allocator CreateAllocator() { return Allocator(); } }; }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index bfaa4860bce..0d22f10f0f2 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -8,6 +8,7 @@ #include +#include "Allocator.hpp" #include "GCScheduler.hpp" #include "Utils.hpp" #include "Types.h" @@ -26,6 +27,8 @@ class NoOpGC : private Pinned { public: class ObjectData {}; + using Allocator = AlignedAllocator; + class ThreadData : private Pinned { public: using ObjectData = NoOpGC::ObjectData; @@ -41,6 +44,8 @@ public: void OnOOM(size_t size) noexcept {} + Allocator CreateAllocator() noexcept { return Allocator(); } + private: }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index c7fc5174e70..2bac0374617 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -193,9 +193,10 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogInfo( {kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(), stableRef, timeRootSetUs - timeSuspendUs); - gc::Mark(std::move(graySet)); + auto markStats = gc::Mark(std::move(graySet)); auto timeMarkUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs); + RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries); + scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); auto timeSweepExtraObjectsUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 43231c4e6eb..15eae8dec68 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -8,6 +8,7 @@ #include +#include "Allocator.hpp" #include "GCScheduler.hpp" #include "ObjectFactory.hpp" #include "Types.h" @@ -48,6 +49,7 @@ public: class ThreadData : private Pinned { public: using ObjectData = SameThreadMarkAndSweep::ObjectData; + using Allocator = AllocatorWithGC; explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} ~ThreadData() = default; @@ -61,6 +63,8 @@ public: void OnOOM(size_t size) noexcept; + Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } + private: void SafePointRegular(size_t weight) noexcept; void SafePointSlowPath(SafepointFlag flag) noexcept; @@ -69,6 +73,8 @@ public: mm::ThreadData& threadData_; }; + using Allocator = ThreadData::Allocator; + SameThreadMarkAndSweep() noexcept; ~SameThreadMarkAndSweep() = default; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 26f1a84ac36..f3e3ed64781 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -53,6 +53,13 @@ public: public: ~Node() = default; + constexpr static std::pair GetSizeAndAlignmentForDataSize(size_t dataSize) noexcept { + size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); + size_t totalAlignment = std::max(alignof(Node), DataAlignment); + size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment); + return std::make_pair(totalSize, totalAlignment); + } + static Node& FromData(void* data) noexcept { constexpr size_t kDataOffset = DataOffset(); Node* node = reinterpret_cast(reinterpret_cast(data) - kDataOffset); @@ -80,9 +87,7 @@ public: Node() noexcept = default; static unique_ptr Create(Allocator& allocator, size_t dataSize) noexcept { - size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); - size_t totalAlignment = std::max(alignof(Node), DataAlignment); - size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment); + auto [totalSize, totalAlignment] = GetSizeAndAlignmentForDataSize(dataSize); RuntimeAssert( DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize, DataOffset()); @@ -392,43 +397,13 @@ private: SpinLock mutex_; }; -class SimpleAllocator { -public: - void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); } - - static void Free(void* instance) noexcept { konanFreeMemory(instance); } -}; - -template -class AllocatorWithGC { -public: - AllocatorWithGC(BaseAllocator base, GC& gc) noexcept : base_(std::move(base)), gc_(gc) {} - - void* Alloc(size_t size, size_t alignment) noexcept { - gc_.SafePointAllocation(size); - if (void* ptr = base_.Alloc(size, alignment)) { - return ptr; - } - // Tell GC that we failed to allocate, and try one more time. - gc_.OnOOM(size); - return base_.Alloc(size, alignment); - } - - static void Free(void* instance) noexcept { BaseAllocator::Free(instance); } - -private: - BaseAllocator base_; - GC& gc_; -}; - } // namespace internal template class ObjectFactory : private Pinned { using GCObjectData = typename GC::ObjectData; using GCThreadData = typename GC::ThreadData; - - using Allocator = internal::AllocatorWithGC; + using Allocator = typename GC::Allocator; struct HeapObjHeader { GCObjectData gcData; @@ -523,13 +498,17 @@ public: typename Storage::Producer::Iterator iterator_; }; - ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept : - producer_(owner.storage_, internal::AllocatorWithGC(internal::SimpleAllocator(), gc)) {} + ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept : producer_(owner.storage_, gc.CreateAllocator()) {} + + static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept { + RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); + size_t allocSize = ObjectAllocatedDataSize(typeInfo); + return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first; + } ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); - size_t allocSize = AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); + size_t allocSize = ObjectAllocatedDataSize(typeInfo); auto& node = producer_.Insert(allocSize); auto* heapObject = new (node.Data()) HeapObjHeader(); auto* object = &heapObject->object; @@ -538,11 +517,15 @@ public: return object; } + static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept { + RuntimeAssert(typeInfo->IsArray(), "Must be an array"); + size_t allocSize = ArrayAllocatedDataSize(typeInfo, count); + return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first; + } + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - uint32_t membersSize = static_cast(-typeInfo->instanceSize_) * count; - // Note: array body is aligned, but for size computation it is enough to align the sum. - size_t allocSize = AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + size_t allocSize = ArrayAllocatedDataSize(typeInfo, count); auto& node = producer_.Insert(allocSize); auto* heapArray = new (node.Data()) HeapArrayHeader(); auto* array = &heapArray->array; @@ -560,6 +543,17 @@ public: void ClearForTests() noexcept { producer_.ClearForTests(); } private: + static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { + size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); + return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); + } + + static size_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { + uint32_t membersSize = static_cast(-typeInfo->instanceSize_) * count; + // Note: array body is aligned, but for size computation it is enough to align the sum. + return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + } + typename Storage::Producer producer_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 74e08a6e69f..bbef5ea24eb 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -12,7 +12,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "Allocator.hpp" #include "FinalizerHooksTestSupport.hpp" +#include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" #include "TestSupport.hpp" #include "Types.h" @@ -23,7 +25,7 @@ using testing::_; namespace { -using SimpleAllocator = mm::internal::SimpleAllocator; +using SimpleAllocator = gc::AlignedAllocator; template using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; @@ -691,86 +693,61 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { EXPECT_THAT(storage.GetSizeUnsafe(), expectedAfter.size()); } -using mm::internal::AllocatorWithGC; - namespace { -class MockAllocator { +class MockAllocator : private Pinned { public: + MockAllocator(); + ~MockAllocator(); + MOCK_METHOD(void*, Alloc, (size_t, size_t)); + MOCK_METHOD(void, Free, (void*)); + + void* DefaultAlloc(size_t size, size_t alignment) { return konanAllocAlignedMemory(size, alignment); } + + void DefaultFree(void* instance) { konanFreeMemory(instance); } }; -class MockAllocatorWrapper { +class GlobalMockAllocator { public: - MockAllocator& operator*() { return *mock_; } + void* Alloc(size_t size, size_t alignment) { + RuntimeAssert(instance_ != nullptr, "Global allocator must be set"); + return instance_->Alloc(size, alignment); + } - void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); } + static void Free(void* instance) { + RuntimeAssert(instance_ != nullptr, "Global allocator must be set"); + instance_->Free(instance); + } + + static void SetMockAllocator(MockAllocator* instance) { + RuntimeAssert(instance != nullptr, "Cannot be null"); + RuntimeAssert(instance_ == nullptr, "No global allocator can be set"); + instance_ = instance; + } + + static void ClearMockAllocator(MockAllocator* instance) { + RuntimeAssert(instance != nullptr, "Cannot be null"); + RuntimeAssert(instance_ == instance, "Allocators must match"); + instance_ = nullptr; + } private: - KStdUniquePtr> mock_ = make_unique>(); + static MockAllocator* instance_; }; -class MockGC { -public: - MOCK_METHOD(void, SafePointAllocation, (size_t)); - MOCK_METHOD(void, OnOOM, (size_t)); -}; - -} // namespace - -TEST(AllocatorWithGCTest, AllocateWithoutOOM) { - constexpr size_t size = 256; - constexpr size_t alignment = 8; - void* nonNull = reinterpret_cast(1); - MockAllocatorWrapper baseAllocator; - testing::StrictMock gc; - { - testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); - EXPECT_CALL(gc, OnOOM(_)).Times(0); - } - AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); - EXPECT_THAT(ptr, nonNull); +MockAllocator::MockAllocator() { + GlobalMockAllocator::SetMockAllocator(this); + ON_CALL(*this, Alloc(_, _)).WillByDefault([this](size_t size, size_t alignment) { return DefaultAlloc(size, alignment); }); + ON_CALL(*this, Free(_)).WillByDefault([this](void* instance) { DefaultFree(instance); }); } -TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { - constexpr size_t size = 256; - constexpr size_t alignment = 8; - void* nonNull = reinterpret_cast(1); - MockAllocatorWrapper baseAllocator; - testing::StrictMock gc; - { - testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); - EXPECT_CALL(gc, OnOOM(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); - } - AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); - EXPECT_THAT(ptr, nonNull); +MockAllocator::~MockAllocator() { + GlobalMockAllocator::ClearMockAllocator(this); } -TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { - constexpr size_t size = 256; - constexpr size_t alignment = 8; - MockAllocatorWrapper baseAllocator; - testing::StrictMock gc; - { - testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); - EXPECT_CALL(gc, OnOOM(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); - } - AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); - EXPECT_THAT(ptr, nullptr); -} - -namespace { +// static +MockAllocator* GlobalMockAllocator::instance_ = nullptr; class GC { public: @@ -778,11 +755,15 @@ public: uint32_t flags = 42; }; + using Allocator = GlobalMockAllocator; + class ThreadData { public: void SafePointAllocation(size_t size) noexcept {} void OnOOM(size_t size) noexcept {} + + Allocator CreateAllocator() noexcept { return Allocator(); } }; }; @@ -801,12 +782,26 @@ struct Payload { } // namespace TEST(ObjectFactoryTest, CreateObject) { + testing::StrictMock allocator; + test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder()}; GC::ThreadData gc; ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); + size_t allocSize = 0; + void* allocAddress = nullptr; + EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + allocSize = size; + allocAddress = allocator.DefaultAlloc(size, alignment); + return allocAddress; + }); auto* object = threadQueue.CreateObject(type.typeInfo()); + testing::Mock::VerifyAndClearExpectations(&allocator); + EXPECT_THAT(allocSize, testing::Gt(type.typeInfo()->instanceSize_)); + EXPECT_THAT(allocAddress, testing::Ne(nullptr)); + EXPECT_THAT(mm::GetAllocatedHeapSize(object), allocSize); + threadQueue.Publish(); auto node = ObjectFactory::NodeRef::From(object); @@ -819,14 +814,30 @@ TEST(ObjectFactoryTest, CreateObject) { EXPECT_THAT(*it, node); ++it; EXPECT_THAT(it, iter.end()); + + EXPECT_CALL(allocator, Free(allocAddress)); } TEST(ObjectFactoryTest, CreateObjectArray) { + testing::StrictMock allocator; + GC::ThreadData gc; ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); + size_t allocSize = 0; + void* allocAddress = nullptr; + EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + allocSize = size; + allocAddress = allocator.DefaultAlloc(size, alignment); + return allocAddress; + }); auto* array = threadQueue.CreateArray(theArrayTypeInfo, 3); + testing::Mock::VerifyAndClearExpectations(&allocator); + EXPECT_THAT(allocSize, testing::Gt(-theArrayTypeInfo->instanceSize_ * 3)); + EXPECT_THAT(allocAddress, testing::Ne(nullptr)); + EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize); + threadQueue.Publish(); auto node = ObjectFactory::NodeRef::From(array); @@ -839,14 +850,30 @@ TEST(ObjectFactoryTest, CreateObjectArray) { EXPECT_THAT(*it, node); ++it; EXPECT_THAT(it, iter.end()); + + EXPECT_CALL(allocator, Free(allocAddress)); } TEST(ObjectFactoryTest, CreateCharArray) { + testing::StrictMock allocator; + GC::ThreadData gc; ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); + size_t allocSize = 0; + void* allocAddress = nullptr; + EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + allocSize = size; + allocAddress = allocator.DefaultAlloc(size, alignment); + return allocAddress; + }); auto* array = threadQueue.CreateArray(theCharArrayTypeInfo, 3); + testing::Mock::VerifyAndClearExpectations(&allocator); + EXPECT_THAT(allocSize, testing::Gt(-theCharArrayTypeInfo->instanceSize_ * 3)); + EXPECT_THAT(allocAddress, testing::Ne(nullptr)); + EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize); + threadQueue.Publish(); auto node = ObjectFactory::NodeRef::From(array); @@ -859,18 +886,24 @@ TEST(ObjectFactoryTest, CreateCharArray) { EXPECT_THAT(*it, node); ++it; EXPECT_THAT(it, iter.end()); + + EXPECT_CALL(allocator, Free(allocAddress)); } TEST(ObjectFactoryTest, Erase) { + testing::StrictMock allocator; + test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder()}; GC::ThreadData gc; ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); + EXPECT_CALL(allocator, Alloc(_, _)).Times(20); for (int i = 0; i < 10; ++i) { threadQueue.CreateObject(objectType.typeInfo()); threadQueue.CreateArray(theArrayTypeInfo, 3); } + testing::Mock::VerifyAndClearExpectations(&allocator); threadQueue.Publish(); @@ -878,7 +911,9 @@ TEST(ObjectFactoryTest, Erase) { auto iter = objectFactory.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->IsArray()) { + EXPECT_CALL(allocator, Free(_)); iter.EraseAndAdvance(it); + testing::Mock::VerifyAndClearExpectations(&allocator); } else { ++it; } @@ -893,19 +928,24 @@ TEST(ObjectFactoryTest, Erase) { } EXPECT_THAT(count, 10); } + EXPECT_CALL(allocator, Free(_)).Times(10); } TEST(ObjectFactoryTest, Move) { + testing::StrictMock allocator; + test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder()}; GC::ThreadData gc; ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); ObjectFactory::FinalizerQueue finalizerQueue; + EXPECT_CALL(allocator, Alloc(_, _)).Times(20); for (int i = 0; i < 10; ++i) { threadQueue.CreateObject(objectType.typeInfo()); threadQueue.CreateArray(theArrayTypeInfo, 3); } + testing::Mock::VerifyAndClearExpectations(&allocator); threadQueue.Publish(); @@ -937,9 +977,13 @@ TEST(ObjectFactoryTest, Move) { } EXPECT_THAT(count, 10); } + + EXPECT_CALL(allocator, Free(_)).Times(20); } TEST(ObjectFactoryTest, RunFinalizers) { + testing::StrictMock allocator; + FinalizerHooksTestSupport finalizerHooks; test_support::TypeInfoHolder objectType{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; @@ -949,9 +993,11 @@ TEST(ObjectFactoryTest, RunFinalizers) { ObjectFactory::FinalizerQueue finalizerQueue; KStdVector objects; + EXPECT_CALL(allocator, Alloc(_, _)).Times(10); for (int i = 0; i < 10; ++i) { objects.push_back(threadQueue.CreateObject(objectType.typeInfo())); } + testing::Mock::VerifyAndClearExpectations(&allocator); threadQueue.Publish(); @@ -968,9 +1014,12 @@ TEST(ObjectFactoryTest, RunFinalizers) { finalizerQueue.Finalize(); // Hooks called before `FinalizerQueue` destructor. testing::Mock::VerifyAndClearExpectations(&finalizerHooks.finalizerHook()); + EXPECT_CALL(allocator, Free(_)).Times(10); } TEST(ObjectFactoryTest, ConcurrentPublish) { + testing::StrictMock allocator; + test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder()}; ObjectFactory objectFactory; constexpr int kThreadCount = kDefaultThreadCount; @@ -980,6 +1029,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { std::mutex expectedMutex; KStdVector expected; + EXPECT_CALL(allocator, Alloc(_, _)).Times(kThreadCount); for (int i = 0; i < kThreadCount; ++i) { threads.emplace_back([&type, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() { GC::ThreadData gc; @@ -998,6 +1048,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { while (readyCount < kThreadCount) { } + testing::Mock::VerifyAndClearExpectations(&allocator); canStart = true; for (auto& t : threads) { t.join(); @@ -1010,4 +1061,5 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { } EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + EXPECT_CALL(allocator, Free(_)).Times(kThreadCount); } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index 0ac28b86767..3e286a5bc53 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -69,3 +69,13 @@ OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, // `ArrayHeader` and `ObjHeader` are expected to be compatible. RETURN_OBJ(reinterpret_cast(array)); } + +size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept { + RuntimeAssert(object->heap(), "Object must be a heap object"); + const auto* typeInfo = object->type_info(); + if (typeInfo->IsArray()) { + return mm::ObjectFactory::ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_); + } else { + return mm::ObjectFactory::ThreadQueue::ObjectAllocatedSize(typeInfo); + } +} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp index 2c20468ae97..9e06fa89b16 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -29,6 +29,9 @@ OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, Obj OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept; +// This does not take into account how much storage did the underlying allocator (malloc/mimalloc) reserved. +size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; + } // namespace mm } // namespace kotlin