diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index b670e741386..87140f7428e 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -370,8 +370,6 @@ bitcode { headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} - testFixtures {} - test {} } onlyIf { target.supportsThreads() } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 1262e83bbd4..9177d7209b6 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -41,8 +41,7 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce return AlignUp(heapArrayHeaderSize + membersSize, kObjectAlignment); } -CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : - heap_(heap), gcScheduler_(gcScheduler), nextFitPage_(nullptr), extraObjectPage_(nullptr) { +CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) { CustomAllocInfo("CustomAllocator::CustomAllocator(heap)"); memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); } @@ -126,7 +125,6 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept { uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes"); - gcScheduler_.OnSafePointAllocation(size); CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size); uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell); uint8_t* ptr; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index ce2a93ec411..c2128317821 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -11,7 +11,6 @@ #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" -#include "GCScheduler.hpp" #include "Heap.hpp" #include "NextFitPage.hpp" #include "Memory.h" @@ -21,7 +20,7 @@ namespace kotlin::alloc { class CustomAllocator { public: - explicit CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept; + explicit CustomAllocator(Heap& heap) noexcept; ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; @@ -49,7 +48,6 @@ private: uint8_t* AllocateInFixedBlockPage(uint32_t cellCount) noexcept; Heap& heap_; - gcScheduler::GCSchedulerThreadData& gcScheduler_; NextFitPage* nextFitPage_; FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; ExtraObjectPage* extraObjectPage_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp index e30008ed63a..1dc0a5f4841 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp @@ -8,7 +8,6 @@ #include "CustomAllocConstants.hpp" #include "CustomAllocator.hpp" -#include "GCScheduler.hpp" #include "Memory.h" #include "gtest/gtest.h" #include "Heap.hpp" @@ -28,9 +27,7 @@ TEST(CustomAllocTest, SmallAllocNonNull) { fakeTypes[i] = {.typeInfo_ = &fakeTypes[i], .instanceSize_ = 8 * i, .flags_ = 0}; } Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); - CustomAllocator ca(heap, schedulerData); + CustomAllocator ca(heap); ObjHeader* obj[N]; for (int i = 1; i < N; ++i) { TypeInfo* type = fakeTypes + i; @@ -43,9 +40,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { const int N = FIXED_BLOCK_PAGE_CELL_COUNT / FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; for (int blocks = MIN_BLOCK_SIZE; blocks < FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blocks) { Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); - CustomAllocator ca(heap, schedulerData); + CustomAllocator ca(heap); TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; uint8_t* first = reinterpret_cast(ca.CreateObject(&fakeType)); for (int i = 1; i < N; ++i) { @@ -58,9 +53,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { TEST(CustomAllocTest, FixedBlockPageThreshold) { Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); - CustomAllocator ca(heap, schedulerData); + CustomAllocator ca(heap); const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10; const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { @@ -71,9 +64,7 @@ TEST(CustomAllocTest, FixedBlockPageThreshold) { TEST(CustomAllocTest, NextFitPageThreshold) { Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); - CustomAllocator ca(heap, schedulerData); + CustomAllocator ca(heap); const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10; const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { @@ -85,11 +76,8 @@ TEST(CustomAllocTest, NextFitPageThreshold) { TEST(CustomAllocTest, TwoAllocatorsDifferentPages) { for (int blocks = MIN_BLOCK_SIZE; blocks < 2000; ++blocks) { Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData1(config, [](auto&) {}); - kotlin::gcScheduler::GCSchedulerThreadData schedulerData2(config, [](auto&) {}); - CustomAllocator ca1(heap, schedulerData1); - CustomAllocator ca2(heap, schedulerData2); + CustomAllocator ca1(heap); + CustomAllocator ca2(heap); TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; uint8_t* obj1 = reinterpret_cast(ca1.CreateObject(&fakeType)); uint8_t* obj2 = reinterpret_cast(ca2.CreateObject(&fakeType)); @@ -102,9 +90,7 @@ using Data = typename kotlin::mm::ExtraObjectData; TEST(CustomAllocTest, AllocExtraObjectNonNullZeroed) { Heap heap; - kotlin::gcScheduler::GCSchedulerConfig config; - kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {}); - CustomAllocator ca(heap, schedulerData); + CustomAllocator ca(heap); for (int i = 1; i < 10; ++i) { uint8_t* obj = reinterpret_cast(ca.CreateExtraObject()); EXPECT_TRUE(obj); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 62122039e8f..2ded2becfaa 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -98,7 +98,8 @@ void* SafeAlloc(uint64_t size) noexcept { konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes: %s. Aborting.\n", size, strerror(errno)); konan::abort(); } - allocatedBytesCounter.fetch_add(static_cast(size), std::memory_order_relaxed); + auto previousSize = allocatedBytesCounter.fetch_add(static_cast(size), std::memory_order_relaxed); + OnMemoryAllocation(previousSize + static_cast(size)); CustomAllocDebug("SafeAlloc(%zu) = %p", static_cast(size), memory); return memory; } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 090b8768cb2..dbe8355f349 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -59,10 +59,6 @@ struct ProcessWeaksTraits { } // namespace -void gc::ConcurrentMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - gcScheduler_.OnSafePointAllocation(size); -} - void gc::ConcurrentMarkAndSweep::ThreadData::Schedule() noexcept { RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); ThreadStateGuard guard(ThreadState::kNative); @@ -187,8 +183,6 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::Mark(gcHandle, markQueue_); mm::WaitForThreadsSuspension(); - auto markStats = gcHandle.getMarked(); - scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes); #ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread @@ -228,6 +222,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); } #endif + scheduler.gcData().UpdateAliveSetBytes(allocatedBytes()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 1297177d3d3..2f85e96b490 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -78,13 +78,9 @@ public: using Allocator = AllocatorWithGC; - explicit ThreadData( - ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : - gc_(gc), threadData_(threadData), gcScheduler_(gcScheduler) {} + explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} ~ThreadData() = default; - void SafePointAllocation(size_t size) noexcept; - void Schedule() noexcept; void ScheduleAndWaitFullGC() noexcept; void ScheduleAndWaitFullGCWithFinalizers() noexcept; @@ -103,7 +99,6 @@ public: friend ConcurrentMarkAndSweep; ConcurrentMarkAndSweep& gc_; mm::ThreadData& threadData_; - gcScheduler::GCSchedulerThreadData& gcScheduler_; std::atomic marking_; BarriersThreadData barriers_; }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 480c1ebabcb..fb1b6b3a970 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -15,8 +15,7 @@ using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 086dcf6be15..10f0657b98e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -44,14 +44,14 @@ private: class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - gc_(gc.impl_->gc(), threadData, gcScheduler), + Impl(GC& gc, mm::ThreadData& threadData) noexcept : + gc_(gc.impl_->gc(), threadData), #ifndef CUSTOM_ALLOCATOR objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { } #else - alloc_(gc.impl_->gc().heap(), gcScheduler) { + alloc_(gc.impl_->gc().heap()) { } #endif diff --git a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp index 858506a02d9..6cc493ccbdc 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp @@ -28,7 +28,6 @@ public: AllocatorWithGC(BaseAllocator base, GCThreadData& gc) noexcept : base_(std::move(base)), gc_(gc) {} void* Alloc(size_t size) noexcept { - gc_.SafePointAllocation(size); if (void* ptr = base_.Alloc(size)) { return ptr; } diff --git a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp index 44de2b131c7..1e64ff7c66b 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp @@ -33,7 +33,6 @@ private: class MockGC { public: - MOCK_METHOD(void, SafePointAllocation, (size_t)); MOCK_METHOD(void, OnOOM, (size_t)); }; @@ -46,7 +45,6 @@ TEST(AllocatorWithGCTest, AllocateWithoutOOM) { testing::StrictMock gc; { testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); EXPECT_CALL(gc, OnOOM(_)).Times(0); } @@ -62,7 +60,6 @@ TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { testing::StrictMock gc; { testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); EXPECT_CALL(gc, OnOOM(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); @@ -78,7 +75,6 @@ TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { testing::StrictMock gc; { testing::InSequence seq; - EXPECT_CALL(gc, SafePointAllocation(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); EXPECT_CALL(gc, OnOOM(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index e00d36f8140..14981c77af3 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -29,7 +29,7 @@ public: public: class Impl; - ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept; + ThreadData(GC& gc, mm::ThreadData& threadData) noexcept; ~ThreadData(); Impl& impl() noexcept { return *impl_; } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index 1990dd39b88..e28f41b5870 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -309,7 +309,6 @@ void GCHandle::marked(kotlin::gc::MarkStats stats) { if (!stat->markStats) { stat->markStats = MarkStats{}; } - stat->markStats->markedSizeBytes += stats.markedSizeBytes; stat->markStats->markedCount += stats.markedCount; } } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 585db9865b0..1df9fc169e7 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -32,7 +32,6 @@ struct SweepStats { struct MarkStats { uint64_t markedCount = 0; - uint64_t markedSizeBytes = 0; }; class GCHandle { @@ -103,10 +102,7 @@ public: explicit GCMarkScope(GCHandle& handle); ~GCMarkScope(); - void addObject(uint64_t objectSize) noexcept { - ++stats_.markedCount; - stats_.markedSizeBytes += objectSize; - } + void addObject() noexcept { ++stats_.markedCount; } }; class GCProcessWeaksScope : GCStageScopeUsTimer, Pinned { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index d3e831189e5..f23ad879cb0 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -81,7 +81,7 @@ void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits:: // Do not schedule RegularWeakReferenceImpl but process it right away. // This will skip markQueue interaction. if (Traits::tryMark(weakReference)) { - markHandle.addObject(mm::GetAllocatedHeapSize(weakReference)); + markHandle.addObject(); // RegularWeakReferenceImpl is empty, but keeping this just in case. Traits::processInMark(markQueue, weakReference); } @@ -94,9 +94,7 @@ template void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept { auto markHandle = handle.mark(); while (ObjHeader* top = Traits::tryDequeue(markQueue)) { - // TODO: Consider moving it to the sweep phase to make this loop more tight. - // This, however, requires care with scheduler interoperation. - markHandle.addObject(mm::GetAllocatedHeapSize(top)); + markHandle.addObject(); Traits::processInMark(markQueue, top); @@ -169,7 +167,6 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T template void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { auto handle = gcHandle.collectThreadRoots(thread); - thread.gcScheduler().OnStoppedForGC(); // TODO: Remove useless mm::ThreadRootSet abstraction. for (auto value : mm::ThreadRootSet(thread)) { if (internal::collectRoot(markQueue, value.object)) { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index 7b78ee307fa..b597da85d64 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -157,19 +157,10 @@ private: ScopedMarkTraits markTraits_; }; -size_t GetObjectsSize(std::initializer_list> objects) { - size_t size = 0; - for (auto& object : objects) { - size += mm::GetAllocatedHeapSize(object.get().header()); - } - return size; -} - #define EXPECT_MARKED(stats, ...) \ do { \ std::initializer_list> objects = {__VA_ARGS__}; \ EXPECT_THAT(stats.markedCount, objects.size()); \ - EXPECT_THAT(stats.markedSizeBytes, GetObjectsSize(objects)); \ EXPECT_THAT(marked(), MarkedMatcher(objects)); \ } while (false) diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 53ae4c6bf06..1c04cfd3bc9 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -15,12 +15,7 @@ using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : -#ifdef CUSTOM_ALLOCATOR - impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} -#else - impl_(std_support::make_unique(gc, threadData)) {} -#endif +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 2e9d1fe3857..1d3e3e93ad1 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -42,8 +42,7 @@ private: class GC::ThreadData::Impl : private Pinned { public: #ifdef CUSTOM_ALLOCATOR - Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - alloc_(gc.impl_->gc().heap(), gcScheduler) {} + Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {} #else Impl(GC& gc, mm::ThreadData& threadData) noexcept : objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 13cb7ab35ee..d3a48790d78 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -39,8 +39,6 @@ public: ThreadData() noexcept {} ~ThreadData() = default; - void SafePointAllocation(size_t size) noexcept {} - void Schedule() noexcept {} void ScheduleAndWaitFullGC() noexcept {} void ScheduleAndWaitFullGCWithFinalizers() noexcept {} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index c6adf7df860..4a8716c70a9 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -15,8 +15,7 @@ using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} gc::GC::ThreadData::~ThreadData() = default; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index cfbfb2e1d93..afed207b219 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -42,10 +42,11 @@ private: class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - gc_(gc.impl_->gc(), threadData, gcScheduler), + Impl(GC& gc, mm::ThreadData& threadData) noexcept : + gc_(gc.impl_->gc(), threadData), #ifdef CUSTOM_ALLOCATOR - alloc_(gc.impl_->gc().heap(), gcScheduler) {} + alloc_(gc.impl_->gc().heap()) { + } #else objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 47d3111a3a5..d1af83126d0 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -58,10 +58,6 @@ struct ProcessWeaksTraits { } // namespace -void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - gcScheduler_.OnSafePointAllocation(size); -} - void gc::SameThreadMarkAndSweep::ThreadData::Schedule() noexcept { RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); ThreadStateGuard guard(ThreadState::kNative); @@ -161,8 +157,6 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::collectRootSet(gcHandle, markQueue_, [](mm::ThreadData&) { return true; }); gc::Mark(gcHandle, markQueue_); - auto markStats = gcHandle.getMarked(); - scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes); gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); @@ -181,6 +175,8 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto finalizerQueue = heap_.Sweep(gcHandle); #endif + scheduler.gcData().UpdateAliveSetBytes(allocatedBytes()); + mm::ResumeThreads(); gcHandle.threadsAreResumed(); state_.finish(epoch); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 90d10f850b3..2ac3f94236a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -79,12 +79,9 @@ public: using ObjectData = SameThreadMarkAndSweep::ObjectData; using Allocator = AllocatorWithGC; - ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : - gc_(gc), gcScheduler_(gcScheduler) {} + ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc) {} ~ThreadData() = default; - void SafePointAllocation(size_t size) noexcept; - void Schedule() noexcept; void ScheduleAndWaitFullGC() noexcept; void ScheduleAndWaitFullGCWithFinalizers() noexcept; @@ -96,7 +93,6 @@ public: private: SameThreadMarkAndSweep& gc_; - gcScheduler::GCSchedulerThreadData& gcScheduler_; }; using Allocator = ThreadData::Allocator; diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp index d9e544c6e35..0c0ad18de05 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -38,14 +38,6 @@ public: RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized"); } - void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override { - heapGrowthController_.OnAllocated(threadData.allocatedBytes()); - if (heapGrowthController_.NeedsGC()) { - RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - scheduleGC_(); - } - } - void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); regularIntervalPacer_.OnPerformFullGC(); @@ -54,6 +46,13 @@ public: void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } + void SetAllocatedBytes(size_t bytes) noexcept override { + if (heapGrowthController_.SetAllocatedBytes(bytes)) { + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + scheduleGC_(); + } + } + private: GCSchedulerConfig& config_; std::function scheduleGC_; diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp index df1ffa531b6..1a140de821f 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp @@ -10,7 +10,6 @@ #include "AppStateTrackingTestSupport.hpp" #include "ClockTestSupport.hpp" -#include "GCSchedulerTestSupport.hpp" #include "SingleThreadExecutor.hpp" #include "TestSupport.hpp" #include "std_support/Vector.hpp" @@ -21,25 +20,18 @@ namespace { class MutatorThread : private Pinned { public: - MutatorThread(gcScheduler::GCSchedulerConfig& config, std::function slowPath) : - executor_([&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); }) {} + explicit MutatorThread(gcScheduler::GCSchedulerData& scheduler) : executor_([&scheduler] { return Context{scheduler}; }) {} - std::future Allocate(size_t bytes) { + std::future SetAllocatedBytes(size_t bytes) { return executor_.execute([&, bytes] { auto& context = executor_.context(); - context.threadDataTestApi.SetAllocatedBytes(bytes); - context.slowPath(context.threadData); + context.scheduler.SetAllocatedBytes(bytes); }); } private: struct Context { - gcScheduler::GCSchedulerThreadData threadData; - gcScheduler::test_support::GCSchedulerThreadDataTestApi threadDataTestApi; - std::function slowPath; - - Context(gcScheduler::GCSchedulerConfig& config, std::function slowPath) : - threadData(config, [](gcScheduler::GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {} + gcScheduler::GCSchedulerData& scheduler; }; SingleThreadExecutor executor_; @@ -51,16 +43,22 @@ public: explicit GCSchedulerDataTestApi(gcScheduler::GCSchedulerConfig& config) : scheduler_(config, scheduleGC_.AsStdFunction()) { mutators_.reserve(MutatorCount); for (int i = 0; i < MutatorCount; ++i) { - mutators_.emplace_back(std_support::make_unique( - config, [this](gcScheduler::GCSchedulerThreadData& threadData) { scheduler_.UpdateFromThreadData(threadData); })); + mutators_.emplace_back(std_support::make_unique(scheduler_)); } } - std::future Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); } + std::future Allocate(int mutator, size_t bytes) { + size_t allocatedBytes = allocatedBytes_.fetch_add(bytes); + allocatedBytes += bytes; + return mutators_[mutator]->SetAllocatedBytes(allocatedBytes); + } void OnPerformFullGC() { scheduler_.OnPerformFullGC(); } - void UpdateAliveSetBytes(size_t bytes) { scheduler_.UpdateAliveSetBytes(bytes); } + void UpdateAliveSetBytes(size_t bytes) { + allocatedBytes_.store(bytes); + scheduler_.UpdateAliveSetBytes(bytes); + } testing::MockFunction& scheduleGC() { return scheduleGC_; } @@ -70,6 +68,7 @@ public: } private: + std::atomic allocatedBytes_ = 0; std_support::vector> mutators_; testing::MockFunction scheduleGC_; gcScheduler::internal::GCSchedulerDataAdaptive scheduler_; diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp index 9e298ace9fe..a9f97d74b2f 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp @@ -22,16 +22,15 @@ class GCSchedulerDataAggressive : public GCSchedulerData { public: GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) { - // Trigger the slowpath on each allocation. - config.allocationThresholdBytes = 1; RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized"); } - void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override { - heapGrowthController_.OnAllocated(threadData.allocatedBytes()); - if (heapGrowthController_.NeedsGC()) { - // Still checking allocations: with a long running loop all safepoints - // might be "met", so that's the only trigger to not run out of memory. + void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); } + void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } + void SetAllocatedBytes(size_t bytes) noexcept override { + // Still checking allocations: with a long running loop all safepoints + // might be "met", so that's the only trigger to not run out of memory. + if (heapGrowthController_.SetAllocatedBytes(bytes)) { RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); scheduleGC_(); } else { @@ -39,9 +38,6 @@ public: } } - void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); } - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - void safePoint() noexcept { if (safePointTracker_.registerCurrentSafePoint(1)) { RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint"); diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp index e84dbf1a7bd..f0d9fff6e19 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp @@ -50,14 +50,9 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { testing::MockFunction scheduleGC; gcScheduler::GCSchedulerConfig config; - gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); - gcScheduler::GCSchedulerThreadData threadSchedulerData( - config, [&scheduler](gcScheduler::GCSchedulerThreadData& data) { scheduler.UpdateFromThreadData(data); }); - - ASSERT_EQ(config.allocationThresholdBytes, 1); - config.autoTune = false; config.targetHeapBytes = 10; + gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); int i = 0; // We trigger GC on the first iteration, when the unique allocation point is faced, @@ -65,7 +60,7 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { EXPECT_CALL(scheduleGC, Call()).WillOnce([&i]() { EXPECT_THAT(i, 0); }).WillOnce([&i]() { EXPECT_THAT(i, 9); }); for (; i < 10; i++) { - threadSchedulerData.OnSafePointAllocation(1); + scheduler.SetAllocatedBytes(i + 1); } testing::Mock::VerifyAndClearExpectations(&scheduleGC); }(); diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp index 5855586bd5c..2143703fdeb 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp @@ -26,9 +26,6 @@ class GCSchedulerData { public: virtual ~GCSchedulerData() = default; - // Called by different mutator threads. - virtual void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept = 0; - // The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC` // when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished. // TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead. @@ -38,48 +35,9 @@ public: // Always called by the GC thread. virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0; -}; -class GCSchedulerThreadData { -public: - explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function slowPath) noexcept : - config_(config), slowPath_(std::move(slowPath)) { - ClearCountersAndUpdateThresholds(); - } - - // Should be called on encountering a safepoint placed by the allocator. - // TODO: Should this even be a safepoint (i.e. a place, where we suspend)? - void OnSafePointAllocation(size_t size) noexcept { - allocatedBytes_ += size; - if (allocatedBytes_ < allocatedBytesThreshold_) { - return; - } - OnSafePointSlowPath(); - } - - void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); } - - size_t allocatedBytes() const noexcept { return allocatedBytes_; } - -private: - friend class test_support::GCSchedulerThreadDataTestApi; - - void OnSafePointSlowPath() noexcept { - slowPath_(*this); - ClearCountersAndUpdateThresholds(); - } - - void ClearCountersAndUpdateThresholds() noexcept { - allocatedBytes_ = 0; - - allocatedBytesThreshold_ = config_.allocationThresholdBytes; - } - - GCSchedulerConfig& config_; - std::function slowPath_; - - size_t allocatedBytes_ = 0; - size_t allocatedBytesThreshold_ = 0; + // Called by different mutator threads. + virtual void SetAllocatedBytes(size_t bytes) noexcept = 0; }; class GCScheduler : private Pinned { @@ -89,10 +47,6 @@ public: GCSchedulerConfig& config() noexcept { return config_; } GCSchedulerData& gcData() noexcept { return *gcData_; } - GCSchedulerThreadData NewThreadData() noexcept { - return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->UpdateFromThreadData(threadData); }); - } - // Should be called on encountering a safepoint. void safePoint() noexcept; diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp index dccb9e7a855..26c67d9ac98 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp @@ -13,8 +13,6 @@ namespace kotlin::gcScheduler { // NOTE: When changing default values, reflect them in GC.kt as well. struct GCSchedulerConfig { - // How many object bytes a thread must allocate to trigger slow path. - std::atomic allocationThresholdBytes = 10 * 1024; std::atomic autoTune = true; // The target interval between collections when Kotlin code is idle. GC will be triggered // by timer no sooner than this value and no later than twice this value since the previous collection. diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp deleted file mode 100644 index abec5b70a21..00000000000 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCScheduler.hpp" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -#include "GCSchedulerTestSupport.hpp" - -using namespace kotlin; - -using testing::_; - -TEST(GCSchedulerThreadDataTest, AllocationSafePoint) { - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - gcScheduler::GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); - }); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kSize); -} - -TEST(GCSchedulerThreadDataTest, ResetByGC) { - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - gcScheduler::GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - - EXPECT_CALL(slowPath, Call(_)).Times(0); - scheduler.OnStoppedForGC(); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); -} - -TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) { - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - gcScheduler::GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - config.allocationThresholdBytes = kAllocationThreshold - kSize; - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - scheduler.OnStoppedForGC(); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); -} - -TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) { - constexpr size_t kSize = 2; - constexpr size_t kCount = 10; - constexpr size_t kAllocationThreshold = kCount * kSize; - testing::MockFunction slowPath; - gcScheduler::GCSchedulerConfig config; - config.allocationThresholdBytes = kAllocationThreshold; - gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction()); - gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler); - - config.allocationThresholdBytes = kAllocationThreshold - kSize; - - EXPECT_CALL(slowPath, Call(_)).Times(0); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold); - }); - scheduler.OnSafePointAllocation(kSize); - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); - - EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) { - EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize); - }); - for (size_t i = 0; i < kCount - 1; ++i) { - scheduler.OnSafePointAllocation(kSize); - } - testing::Mock::VerifyAndClearExpectations(&slowPath); - EXPECT_THAT(scheduler.allocatedBytes(), 0); -} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp deleted file mode 100644 index 255d187b666..00000000000 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCSchedulerTestSupport.hpp" diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp deleted file mode 100644 index 25fe6fa00af..00000000000 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#include "GCScheduler.hpp" - -namespace kotlin::gcScheduler::test_support { - -class GCSchedulerThreadDataTestApi : private Pinned { -public: - explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {} - - void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; } - -private: - GCSchedulerThreadData& scheduler_; -}; - -} // namespace kotlin::gcScheduler::test_support diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp index 259a13d39e0..704e526b674 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp @@ -17,44 +17,39 @@ namespace kotlin::gcScheduler::internal { class HeapGrowthController { public: - explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : config_(config) {} + explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : + config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) {} // Called by the mutators. - void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; } - - // Called by the GC thread. - void OnPerformFullGC() noexcept { allocatedBytes_ = 0; } + // Returns true if needs GC. + bool SetAllocatedBytes(size_t totalAllocatedBytes) noexcept { return totalAllocatedBytes >= targetHeapBytes_; } // Called by the GC thread. void UpdateAliveSetBytes(size_t bytes) noexcept { - lastAliveSetBytes_ = bytes; - if (config_.autoTune.load()) { double targetHeapBytes = static_cast(bytes) / config_.targetHeapUtilization; if (!std::isfinite(targetHeapBytes)) { // This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything. return; } - double minHeapBytes = static_cast(config_.minHeapBytes.load()); - double maxHeapBytes = static_cast(config_.maxHeapBytes.load()); + double minHeapBytes = static_cast(config_.minHeapBytes.load(std::memory_order_relaxed)); + double maxHeapBytes = static_cast(config_.maxHeapBytes.load(std::memory_order_relaxed)); targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes); - config_.targetHeapBytes = static_cast(targetHeapBytes); + config_.targetHeapBytes.store(static_cast(targetHeapBytes), std::memory_order_relaxed); + targetHeapBytes_ = static_cast(targetHeapBytes); + } else { + targetHeapBytes_ = config_.targetHeapBytes.load(std::memory_order_relaxed); } } - // Called by the mutators. - bool NeedsGC() const noexcept { - uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load(); - uint64_t targetHeapBytes = config_.targetHeapBytes; - return currentHeapBytes >= targetHeapBytes; + void OnPerformFullGC() noexcept { + // TODO: Need to protect against mutators that can overrun the GC thread. + targetHeapBytes_ = std::numeric_limits::max(); } private: GCSchedulerConfig& config_; - // Updated by both the mutators and the GC thread. - std::atomic allocatedBytes_ = 0; - // Updated by the GC thread, read by the mutators. - std::atomic lastAliveSetBytes_ = 0; + size_t targetHeapBytes_ = 0; }; } // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp index c1bf97ce2ea..4f1d373b99b 100644 --- a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp @@ -15,9 +15,9 @@ class GCSchedulerDataManual : public GCSchedulerData { public: GCSchedulerDataManual() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); } - void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override {} void OnPerformFullGC() noexcept override {} void UpdateAliveSetBytes(size_t bytes) noexcept override {} + void SetAllocatedBytes(size_t bytes) noexcept override {} }; } // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 730a5273211..cb0c1636058 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3786,3 +3786,5 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {} bool kotlin::FinalizersThreadIsRunning() noexcept { return false; } + +void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {} diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index cdf95520d35..89e24b5e977 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -553,6 +553,8 @@ extern const bool kSupportsMultipleMutators; void StartFinalizerThreadIfNeeded() noexcept; bool FinalizersThreadIsRunning() noexcept; +void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept; + } // namespace kotlin RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object); diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt index 4bd61fbed32..1136e2b7345 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt @@ -135,14 +135,12 @@ public object GC { } /** - * How many bytes a thread can allocate before informing the GC scheduler. - * - * Default: 10 * 1024 + * Deprecated and unused. * * Legacy MM: GC allocation threshold, controlling how many bytes allocated since last * collection will trigger new GC. * - * Default: (legacy MM) 8 * 1024 * 1024 + * Default: 8 * 1024 * 1024 * * @throws [IllegalArgumentException] when value is not positive. */ diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h index 9b04b13f5e9..100f155f590 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h @@ -375,6 +375,8 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, #if KONAN_MI_MALLOC mi_decl_nodiscard mi_decl_export size_t mi_allocated_size(void) mi_attr_noexcept; +// This hook must be implemented by the embedder. `allocated_size` is the new size of the heap. +mi_decl_export void mi_hook_allocation(size_t allocated_size) mi_attr_noexcept; #endif #ifdef __cplusplus diff --git a/kotlin-native/runtime/src/mimalloc/c/region.c b/kotlin-native/runtime/src/mimalloc/c/region.c index 4d9d6fed28c..974bcf3a50c 100644 --- a/kotlin-native/runtime/src/mimalloc/c/region.c +++ b/kotlin-native/runtime/src/mimalloc/c/region.c @@ -387,7 +387,8 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed #endif #if KONAN_MI_MALLOC - mi_atomic_add_relaxed(&allocated_size, size); + size_t prev_allocated_size = mi_atomic_add_relaxed(&allocated_size, size); + mi_hook_allocation(prev_allocated_size + size); #endif } return p; diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index f00d0971288..51a92f94a04 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -358,12 +358,14 @@ extern "C" int64_t Kotlin_native_internal_GC_getCollectCyclesThreshold(ObjHeader } extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, int64_t value) { - RuntimeAssert(value > 0, "Must be handled by the caller"); - mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = value; + // TODO: Remove when legacy MM is gone. + // Nothing to do } extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) { - return mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load(); + // TODO: Remove when legacy MM is gone. + // Nothing to do + return 0; } extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) { @@ -619,3 +621,7 @@ RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHea RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakRef) { mm::disposeRegularWeakReferenceImpl(weakRef); } + +void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept { + mm::GlobalData::Instance().gcScheduler().gcData().SetAllocatedBytes(totalAllocatedBytes); +} diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 5a158d81713..8a59cdebfd7 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -11,7 +11,6 @@ #include "GlobalData.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" -#include "GCScheduler.hpp" #include "ObjectFactory.hpp" #include "ShadowStack.hpp" #include "SpecialRefRegistry.hpp" @@ -33,8 +32,7 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), - gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), - gc_(GlobalData::Instance().gc(), gcScheduler_, *this), + gc_(GlobalData::Instance().gc(), *this), suspensionData_(ThreadState::kNative, *this) {} ~ThreadData() = default; @@ -55,8 +53,6 @@ public: std_support::vector>& initializingSingletons() noexcept { return initializingSingletons_; } - gcScheduler::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } - gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -80,7 +76,6 @@ private: ThreadLocalStorage tls_; SpecialRefRegistry::ThreadQueue specialRefRegistry_; ShadowStack shadowStack_; - gcScheduler::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_; std_support::vector> initializingSingletons_; ThreadSuspensionData suspensionData_; diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp index 6627c96710a..6551b15c23b 100644 --- a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp @@ -72,3 +72,7 @@ void kotlin::compactObjectPoolInMainThread() noexcept { size_t kotlin::allocatedBytes() noexcept { return mi_allocated_size(); } + +extern "C" void mi_hook_allocation(size_t allocated_size) mi_attr_noexcept { + OnMemoryAllocation(allocated_size); +} diff --git a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp index 7e91582b2a5..dd9b885cbcf 100644 --- a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp @@ -9,6 +9,8 @@ #include #endif +#include "Memory.h" + #if KONAN_INTERNAL_DLMALLOC extern "C" void* dlcalloc(size_t, size_t); extern "C" void dlfree(void*); @@ -37,13 +39,17 @@ size_t allocatedBytesCounter = 0; void kotlin::initObjectPool() noexcept {} void* kotlin::allocateInObjectPool(size_t size) noexcept { + // TODO: Check that alignment to kObjectAlignment is satisfied. + void* result = callocImpl(1, size); #ifndef KONAN_NO_THREADS - allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed); + auto newSize = allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed); + newSize += size; #else allocatedBytesCounter += size; + auto newSize = allocatedBytesCounter; #endif - // TODO: Check that alignment to kObjectAlignment is satisfied. - return callocImpl(1, size); + OnMemoryAllocation(newSize); + return result; } void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {