azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Add hard memory boundary ^KT-54727

Browse Source
This commit is contained in:
Alexander Shabalin
2023-06-28 10:38:03 +02:00
committed by Space Team
parent 220ecc4788
commit 576f6642f9
35 changed files with 1403 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/backend.native/tests/build.gradle
+1 -2
View File
@@ -3150,8 +3150,7 @@ standaloneTest("stress_gc_allocations") {
(project.testTarget != "watchos_simulator_arm64") &&
!isNoopGC &&
!isAggressiveGC && // TODO: Investigate why too slow
!runtimeAssertionsPanic && // New allocator with assertions makes this test very slow
(project.testTarget != "mingw_x64") // TODO: Fix on mingw.
!runtimeAssertionsPanic // New allocator with assertions makes this test very slow
source = "runtime/memory/stress_gc_allocations.kt"
flags = ['-tr', '-opt-in=kotlin.native.internal.InternalForKotlinNative']
}
kotlin-native/backend.native/tests/runtime/memory/stress_gc_allocations.kt
+1
View File
@@ -62,6 +62,7 @@ fun test() {
if (Platform.memoryModel == MemoryModel.EXPERIMENTAL) {
kotlin.native.runtime.GC.autotune = false
kotlin.native.runtime.GC.targetHeapBytes = retainLimit
kotlin.native.runtime.GC.pauseOnTargetHeapOverflow = true
}
// On Linux, the child process might immediately commit the same amount of memory as the parent.
kotlin-native/runtime/build.gradle.kts
+1
View File
@@ -370,6 +370,7 @@ bitcode {
headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
sourceSets {
main {}
test {}
}
onlyIf { target.supportsThreads() }
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+4 -23
View File
@@ -59,29 +59,10 @@ struct ProcessWeaksTraits {
} // namespace
void gc::ConcurrentMarkAndSweep::ThreadData::Schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
gc_.state_.schedule();
}
void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
auto scheduled_epoch = gc_.state_.schedule();
gc_.state_.waitEpochFinished(scheduled_epoch);
}
void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
auto scheduled_epoch = gc_.state_.schedule();
gc_.state_.waitEpochFinalized(scheduled_epoch);
}
void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
ScheduleAndWaitFullGC();
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
@@ -173,7 +154,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
#endif
auto& scheduler = gcScheduler_;
scheduler.gcData().OnPerformFullGC();
scheduler.onGCStart();
state_.start(epoch);
@@ -222,7 +203,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
}
#endif
scheduler.gcData().UpdateAliveSetBytes(allocatedBytes());
scheduler.onGCFinish(epoch, allocatedBytes());
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+2 -8
View File
@@ -78,13 +78,9 @@ public:
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : threadData_(threadData) {}
~ThreadData() = default;
void Schedule() noexcept;
void ScheduleAndWaitFullGC() noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept;
void OnOOM(size_t size) noexcept;
void OnSuspendForGC() noexcept;
@@ -97,7 +93,6 @@ public:
private:
friend ConcurrentMarkAndSweep;
ConcurrentMarkAndSweep& gc_;
mm::ThreadData& threadData_;
std::atomic<bool> marking_;
BarriersThreadData barriers_;
@@ -135,8 +130,7 @@ public:
alloc::Heap& heap() noexcept { return heap_; }
#endif
int64_t Schedule() noexcept { return state_.schedule(); }
void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); }
GCStateHolder& state() noexcept { return state_; }
private:
void PerformFullGC(int64_t epoch) noexcept;
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+19 -19
View File
@@ -250,7 +250,7 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -296,7 +296,7 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -320,7 +320,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) {
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -337,7 +337,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -357,7 +357,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -374,7 +374,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
@@ -407,7 +407,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -456,7 +456,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -507,7 +507,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -540,7 +540,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false);
@@ -584,8 +584,8 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -615,7 +615,7 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) {
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
@@ -635,7 +635,7 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) {
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
@@ -742,7 +742,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsCollect) {
}));
}
mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
gcDone.store(true, std::memory_order_relaxed);
for (auto& future : gcFutures) {
@@ -792,7 +792,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsAllCollect) {
for (auto& mutator : mutators) {
gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
// If GC starts before all thread executed line above, two gc will be run
// So we temporary switch threads to native state and then return them back after all GC runs are done
SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative);
@@ -939,7 +939,7 @@ TEST_P(ConcurrentMarkAndSweepTest, CrossThreadReference) {
}));
}
mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
gcDone.store(true, std::memory_order_relaxed);
for (auto& future : gcFutures) {
@@ -1142,7 +1142,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
global1->field1 = object1_local.header();
while (weak.load() == nullptr)
;
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
@@ -1153,7 +1153,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
global1->field1 = nullptr;
EXPECT_CALL(finalizerHook(), Call(weak.load()->header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header()));
done = true;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+6 -14
View File
@@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
gc::GC::ThreadData::~ThreadData() = default;
void gc::GC::ThreadData::Schedule() noexcept {
impl_->gc().Schedule();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
impl_->gc().ScheduleAndWaitFullGC();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
}
void gc::GC::ThreadData::Publish() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->extraObjectDataFactoryThreadQueue().Publish();
@@ -144,11 +132,15 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe
}
int64_t gc::GC::Schedule() noexcept {
return impl_->gc().Schedule();
return impl_->gc().state().schedule();
}
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
impl_->gc().WaitFinalized(epoch);
impl_->gc().state().waitEpochFinalized(epoch);
}
bool gc::isMarked(ObjHeader* object) noexcept {
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+3 -6
View File
@@ -6,6 +6,7 @@
#pragma once
#include <atomic>
#include <cstdint>
#include "ExtraObjectData.hpp"
#include "GCScheduler.hpp"
@@ -34,10 +35,6 @@ public:
Impl& impl() noexcept { return *impl_; }
void Schedule() noexcept;
void ScheduleAndWaitFullGC() noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept;
void Publish() noexcept;
void ClearForTests() noexcept;
@@ -73,10 +70,10 @@ public:
static void processArrayInMark(void* state, ArrayHeader* array) noexcept;
static void processFieldInMark(void* state, ObjHeader* field) noexcept;
// TODO: These should be moved into the scheduler.
// TODO: These should exist only in the scheduler.
int64_t Schedule() noexcept;
void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept { WaitFinalizers(Schedule()); }
static const size_t objectDataSize;
static bool SweepObject(void* objectData) noexcept;
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+3 -12
View File
@@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
gc::GC::ThreadData::~ThreadData() = default;
void gc::GC::ThreadData::Schedule() noexcept {
impl_->gc().Schedule();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
impl_->gc().ScheduleAndWaitFullGC();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
}
void gc::GC::ThreadData::Publish() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->extraObjectDataFactoryThreadQueue().Publish();
@@ -131,6 +119,9 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe
int64_t gc::GC::Schedule() noexcept {
return 0;
}
void gc::GC::WaitFinished(int64_t epoch) noexcept {}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {}
bool gc::isMarked(ObjHeader* object) noexcept {
kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp
-4
View File
@@ -39,10 +39,6 @@ public:
ThreadData() noexcept {}
~ThreadData() = default;
void Schedule() noexcept {}
void ScheduleAndWaitFullGC() noexcept {}
void ScheduleAndWaitFullGCWithFinalizers() noexcept {}
void OnOOM(size_t size) noexcept {}
Allocator CreateAllocator() noexcept { return Allocator(); }
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+6 -14
View File
@@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
gc::GC::ThreadData::~ThreadData() = default;
void gc::GC::ThreadData::Schedule() noexcept {
impl_->gc().Schedule();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
impl_->gc().ScheduleAndWaitFullGC();
}
void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
}
void gc::GC::ThreadData::Publish() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->extraObjectDataFactoryThreadQueue().Publish();
@@ -140,11 +128,15 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe
}
int64_t gc::GC::Schedule() noexcept {
return impl_->gc().Schedule();
return impl_->gc().state().schedule();
}
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
impl_->gc().WaitFinalized(epoch);
impl_->gc().state().waitEpochFinalized(epoch);
}
bool gc::isMarked(ObjHeader* object) noexcept {
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+4 -23
View File
@@ -58,29 +58,10 @@ struct ProcessWeaksTraits {
} // namespace
void gc::SameThreadMarkAndSweep::ThreadData::Schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
gc_.state_.schedule();
}
void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
auto scheduled_epoch = gc_.state_.schedule();
gc_.state_.waitEpochFinished(scheduled_epoch);
}
void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
auto scheduled_epoch = gc_.state_.schedule();
gc_.state_.waitEpochFinalized(scheduled_epoch);
}
void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
ScheduleAndWaitFullGC();
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
#ifdef CUSTOM_ALLOCATOR
@@ -142,7 +123,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gcHandle.threadsAreSuspended();
auto& scheduler = gcScheduler_;
scheduler.gcData().OnPerformFullGC();
scheduler.onGCStart();
state_.start(epoch);
@@ -175,7 +156,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
auto finalizerQueue = heap_.Sweep(gcHandle);
#endif
scheduler.gcData().UpdateAliveSetBytes(allocatedBytes());
scheduler.onGCFinish(epoch, allocatedBytes());
mm::ResumeThreads();
gcHandle.threadsAreResumed();
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+2 -9
View File
@@ -79,20 +79,14 @@ public:
using ObjectData = SameThreadMarkAndSweep::ObjectData;
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc) {}
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {}
~ThreadData() = default;
void Schedule() noexcept;
void ScheduleAndWaitFullGC() noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept;
void OnOOM(size_t size) noexcept;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
private:
SameThreadMarkAndSweep& gc_;
};
using Allocator = ThreadData::Allocator;
@@ -118,8 +112,7 @@ public:
void StopFinalizerThreadIfRunning() noexcept;
bool FinalizersThreadIsRunning() noexcept;
int64_t Schedule() noexcept { return state_.schedule(); }
void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); }
GCStateHolder& state() noexcept { return state_; }
#ifdef CUSTOM_ALLOCATOR
alloc::Heap& heap() noexcept { return heap_; }
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+19 -19
View File
@@ -246,7 +246,7 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -292,7 +292,7 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -316,7 +316,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) {
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -333,7 +333,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -353,7 +353,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
});
@@ -370,7 +370,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
@@ -403,7 +403,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -452,7 +452,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -503,7 +503,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -536,7 +536,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false);
@@ -580,8 +580,8 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(
Alive(threadData),
@@ -611,7 +611,7 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) {
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
@@ -631,7 +631,7 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) {
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
@@ -738,7 +738,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) {
}));
}
mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
gcDone.store(true, std::memory_order_relaxed);
for (auto& future : gcFutures) {
@@ -788,7 +788,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAllCollect) {
for (auto& mutator : mutators) {
gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
// If GC starts before all thread executed line above, two gc will be run
// So we temporary switch threads to native state and then return them back after all GC runs are done
SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative);
@@ -935,7 +935,7 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) {
}));
}
mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
gcDone.store(true, std::memory_order_relaxed);
for (auto& future : gcFutures) {
@@ -1138,7 +1138,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
global1->field1 = object1_local.header();
while (weak.load() == nullptr)
;
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
@@ -1149,7 +1149,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
global1->field1 = nullptr;
EXPECT_CALL(finalizerHook(), Call(weak.load()->header()));
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header()));
done = true;
kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp
+50 -5
View File
@@ -13,12 +13,57 @@
using namespace kotlin;
gcScheduler::GCScheduler::GCScheduler() noexcept :
gcData_(std_support::make_unique<internal::GCSchedulerDataAdaptive<steady_clock>>(config_, []() noexcept {
gcScheduler::GCScheduler::ThreadData::Impl::Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept :
scheduler_(scheduler.impl().impl()), mutatorAssists_(scheduler_.mutatorAssists(), thread) {}
gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler& scheduler, mm::ThreadData& thread) noexcept :
impl_(std_support::make_unique<Impl>(scheduler, thread)) {}
gcScheduler::GCScheduler::ThreadData::~ThreadData() = default;
gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept :
impl_(config, []() noexcept {
// This call acquires a lock, but the lock are always short-lived,
// so we ignore thread state switching to avoid recursive safe points.
CallsCheckerIgnoreGuard guard;
mm::GlobalData::Instance().gc().Schedule();
})) {}
return mm::GlobalData::Instance().gc().Schedule();
}) {}
ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept {}
gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique<Impl>(config_)) {}
gcScheduler::GCScheduler::~GCScheduler() = default;
ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept {
impl().mutatorAssists().safePoint();
}
void gcScheduler::GCScheduler::schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
impl().impl().schedule();
}
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
mm::GlobalData::Instance().gc().WaitFinished(epoch);
}
void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
impl().impl().setAllocatedBytes(bytes);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {
impl().impl().onGCStart();
}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {
impl().impl().onGCFinish(epoch, aliveBytes);
}
kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp
+73 -14
View File
@@ -12,15 +12,38 @@
#include "GlobalData.hpp"
#include "HeapGrowthController.hpp"
#include "Logging.hpp"
#include "MutatorAssists.hpp"
#include "RegularIntervalPacer.hpp"
#include "RepeatedTimer.hpp"
#include "SafePoint.hpp"
#include "ThreadData.hpp"
namespace kotlin::gcScheduler::internal {
namespace kotlin::gcScheduler {
namespace internal {
template <typename Clock>
class GCSchedulerDataAdaptive;
}
class GCScheduler::ThreadData::Impl : private Pinned {
public:
Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept;
internal::GCSchedulerDataAdaptive<steady_clock>& scheduler() noexcept { return scheduler_; }
internal::MutatorAssists::ThreadData& mutatorAssists() noexcept { return mutatorAssists_; }
private:
internal::GCSchedulerDataAdaptive<steady_clock>& scheduler_;
internal::MutatorAssists::ThreadData mutatorAssists_;
};
namespace internal {
template <typename Clock>
class GCSchedulerDataAdaptive : public GCSchedulerData {
class GCSchedulerDataAdaptive {
public:
GCSchedulerDataAdaptive(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
GCSchedulerDataAdaptive(GCSchedulerConfig& config, std::function<int64_t()> scheduleGC) noexcept :
config_(config),
scheduleGC_(std::move(scheduleGC)),
appStateTracking_(mm::GlobalData::Instance().appStateTracking()),
@@ -32,34 +55,70 @@ public:
}
if (regularIntervalPacer_.NeedsGC()) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by timer");
scheduleGC_();
schedule();
}
}) {
RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized");
}
void OnPerformFullGC() noexcept override {
heapGrowthController_.OnPerformFullGC();
void onGCStart() noexcept {
regularIntervalPacer_.OnPerformFullGC();
timer_.restart(config_.regularGcInterval());
}
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
void SetAllocatedBytes(size_t bytes) noexcept override {
if (heapGrowthController_.SetAllocatedBytes(bytes)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
scheduleGC_();
void setAllocatedBytes(size_t bytes) noexcept {
auto boundary = heapGrowthController_.boundaryForHeapSize(bytes);
switch (boundary) {
case HeapGrowthController::MemoryBoundary::kNone:
return;
case HeapGrowthController::MemoryBoundary::kTrigger:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
schedule();
return;
case HeapGrowthController::MemoryBoundary::kTarget:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
auto epoch = schedule();
RuntimeLogWarning({kTagGC}, "Pausing the mutators");
mutatorAssists_.requestAssists(epoch);
return;
}
}
void onGCFinish(int64_t epoch, size_t bytes) noexcept {
heapGrowthController_.updateBoundaries(bytes);
// Must wait for all mutators to be released. GC thread cannot continue.
// This is the contract between GC and mutators. With regular native state
// each mutator must check that GC is not doing something. Here GC must check
// that each mutator has done all it needs.
mutatorAssists_.completeEpoch(epoch, [](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& {
return threadData.gcScheduler().impl().mutatorAssists();
});
}
int64_t schedule() noexcept { return scheduleGC_(); }
MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; }
private:
GCSchedulerConfig& config_;
std::function<void()> scheduleGC_;
std::function<int64_t()> scheduleGC_;
mm::AppStateTracking& appStateTracking_;
HeapGrowthController heapGrowthController_;
RegularIntervalPacer<Clock> regularIntervalPacer_;
RepeatedTimer<Clock> timer_;
MutatorAssists mutatorAssists_;
};
} // namespace kotlin::gcScheduler::internal
} // namespace internal
class GCScheduler::Impl : private Pinned {
public:
explicit Impl(GCSchedulerConfig& config) noexcept;
internal::GCSchedulerDataAdaptive<steady_clock>& impl() noexcept { return impl_; }
private:
internal::GCSchedulerDataAdaptive<steady_clock> impl_;
};
} // namespace kotlin::gcScheduler
kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp
+71 -25
View File
@@ -20,18 +20,19 @@ namespace {
class MutatorThread : private Pinned {
public:
explicit MutatorThread(gcScheduler::GCSchedulerData& scheduler) : executor_([&scheduler] { return Context{scheduler}; }) {}
explicit MutatorThread(gcScheduler::internal::GCSchedulerDataAdaptive<test_support::manual_clock>& scheduler) :
executor_([&scheduler] { return Context{scheduler}; }) {}
std::future<void> SetAllocatedBytes(size_t bytes) {
return executor_.execute([&, bytes] {
auto& context = executor_.context();
context.scheduler.SetAllocatedBytes(bytes);
context.scheduler.setAllocatedBytes(bytes);
});
}
private:
struct Context {
gcScheduler::GCSchedulerData& scheduler;
gcScheduler::internal::GCSchedulerDataAdaptive<test_support::manual_clock>& scheduler;
};
SingleThreadExecutor<Context> executor_;
@@ -53,24 +54,26 @@ public:
return mutators_[mutator]->SetAllocatedBytes(allocatedBytes);
}
void OnPerformFullGC() { scheduler_.OnPerformFullGC(); }
void OnPerformFullGC() { scheduler_.onGCStart(); }
void UpdateAliveSetBytes(size_t bytes) {
void onGCFinish(int64_t epoch, size_t bytes) {
allocatedBytes_.store(bytes);
scheduler_.UpdateAliveSetBytes(bytes);
scheduler_.onGCFinish(epoch, bytes);
}
testing::MockFunction<void()>& scheduleGC() { return scheduleGC_; }
testing::MockFunction<int64_t()>& scheduleGC() { return scheduleGC_; }
template <typename Duration>
void advance_time(Duration duration) {
test_support::manual_clock::sleep_for(duration);
}
int64_t assistsRequested() noexcept { return scheduler_.mutatorAssists().assistsRequested(std::memory_order_relaxed); }
private:
std::atomic<size_t> allocatedBytes_ = 0;
std_support::vector<std_support::unique_ptr<MutatorThread>> mutators_;
testing::MockFunction<void()> scheduleGC_;
testing::MockFunction<int64_t()> scheduleGC_;
gcScheduler::internal::GCSchedulerDataAdaptive<test_support::manual_clock> scheduler_;
};
@@ -88,34 +91,77 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTargetHeapReached) {
config.regularGcIntervalMicroseconds = 10;
config.autoTune = false;
config.targetHeapBytes = (mutatorsCount + 1) * 10;
config.heapTriggerCoefficient = 0.9;
config.setMutatorAssists(true);
GCSchedulerDataTestApi<mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
std_support::vector<std::future<void>> futures;
for (int i = 0; i < mutatorsCount; ++i) {
futures.push_back(schedulerTestApi.Allocate(i, 10));
futures.push_back(schedulerTestApi.Allocate(i, 9));
}
for (auto& future : futures) {
future.get();
}
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
schedulerTestApi.Allocate(0, 10).get();
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
schedulerTestApi.Allocate(0, 9).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_THAT(schedulerTestApi.assistsRequested(), 0);
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
schedulerTestApi.onGCFinish(1, 0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2));
schedulerTestApi.Allocate(0, mutatorsCount * 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_THAT(schedulerTestApi.assistsRequested(), 2);
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
schedulerTestApi.onGCFinish(2, 0);
}
TEST_F(AdaptiveSchedulerTest, CollectOnTargetHeapReachedWithoutAssists) {
constexpr int mutatorsCount = kDefaultThreadCount;
gcScheduler::GCSchedulerConfig config;
config.regularGcIntervalMicroseconds = 10;
config.autoTune = false;
config.targetHeapBytes = (mutatorsCount + 1) * 10;
config.heapTriggerCoefficient = 0.9;
config.setMutatorAssists(false);
GCSchedulerDataTestApi<mutatorsCount> schedulerTestApi(config);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
std_support::vector<std::future<void>> futures;
for (int i = 0; i < mutatorsCount; ++i) {
futures.push_back(schedulerTestApi.Allocate(i, 9));
}
for (auto& future : futures) {
future.get();
}
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1));
schedulerTestApi.Allocate(0, 9).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_THAT(schedulerTestApi.assistsRequested(), 0);
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.onGCFinish(1, 0);
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2));
schedulerTestApi.Allocate(0, mutatorsCount * 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
EXPECT_THAT(schedulerTestApi.assistsRequested(), 0);
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.onGCFinish(2, 0);
}
TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) {
@@ -135,7 +181,7 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) {
test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
schedulerTestApi.onGCFinish(1, 0);
}
TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) {
@@ -155,7 +201,7 @@ TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) {
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
schedulerTestApi.onGCFinish(1, 0);
// pending should restart to be 10us since the previous collection without scheduling another GC.
EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0);
@@ -176,7 +222,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotTuneTargetHeap) {
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
schedulerTestApi.onGCFinish(1, 10);
EXPECT_THAT(config.targetHeapBytes.load(), 10);
}
@@ -197,7 +243,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(10);
schedulerTestApi.onGCFinish(1, 10);
EXPECT_THAT(config.targetHeapBytes.load(), 20);
@@ -206,7 +252,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
schedulerTestApi.Allocate(0, 10).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(20);
schedulerTestApi.onGCFinish(2, 20);
EXPECT_THAT(config.targetHeapBytes.load(), 40);
@@ -215,7 +261,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
schedulerTestApi.Allocate(0, 40).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
schedulerTestApi.onGCFinish(3, 60);
// But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down
EXPECT_THAT(config.targetHeapBytes.load(), 50);
@@ -225,7 +271,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
schedulerTestApi.Allocate(0, 0).get();
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(60);
schedulerTestApi.onGCFinish(4, 60);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
@@ -234,7 +280,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 40
schedulerTestApi.UpdateAliveSetBytes(40);
schedulerTestApi.onGCFinish(5, 40);
EXPECT_THAT(config.targetHeapBytes.load(), 50);
@@ -244,7 +290,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) {
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
// Dropping to 1
schedulerTestApi.UpdateAliveSetBytes(1);
schedulerTestApi.onGCFinish(6, 1);
// But the minimum is set to 5.
EXPECT_THAT(config.targetHeapBytes.load(), 5);
@@ -283,5 +329,5 @@ TEST_F(AdaptiveSchedulerTest, DoNotCollectOnTimerInBackground) {
test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10));
testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC());
schedulerTestApi.OnPerformFullGC();
schedulerTestApi.UpdateAliveSetBytes(0);
schedulerTestApi.onGCFinish(1, 0);
}
kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp
+48 -6
View File
@@ -13,14 +13,56 @@
using namespace kotlin;
gcScheduler::GCScheduler::GCScheduler() noexcept :
gcData_(std_support::make_unique<internal::GCSchedulerDataAggressive>(config_, []() noexcept {
gcScheduler::GCScheduler::ThreadData::Impl::Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept :
scheduler_(scheduler.impl().impl()), mutatorAssists_(scheduler_.mutatorAssists(), thread) {}
gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler& gcScheduler, mm::ThreadData& thread) noexcept :
impl_(std_support::make_unique<Impl>(gcScheduler, thread)) {}
gcScheduler::GCScheduler::ThreadData::~ThreadData() = default;
gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept :
impl_(config, []() noexcept {
// This call acquires a lock, but the lock are always short-lived,
// so we ignore thread state switching to avoid recursive safe points.
CallsCheckerIgnoreGuard guard;
mm::GlobalData::Instance().gc().Schedule();
})) {}
return mm::GlobalData::Instance().gc().Schedule();
}) {}
ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept {
static_cast<internal::GCSchedulerDataAggressive&>(gcData()).safePoint();
gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique<Impl>(config_)) {}
gcScheduler::GCScheduler::~GCScheduler() = default;
ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept {
impl().mutatorAssists().safePoint();
impl().scheduler().safePoint();
}
void gcScheduler::GCScheduler::schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
impl().impl().schedule();
}
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
mm::GlobalData::Instance().gc().WaitFinished(epoch);
}
void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
impl().impl().setAllocatedBytes(bytes);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {
impl().impl().onGCFinish(epoch, aliveBytes);
}
kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp
+71 -14
View File
@@ -12,44 +12,101 @@
#include "GCSchedulerConfig.hpp"
#include "HeapGrowthController.hpp"
#include "Logging.hpp"
#include "MutatorAssists.hpp"
#include "SafePoint.hpp"
#include "SafePointTracker.hpp"
#include "ThreadData.hpp"
namespace kotlin::gcScheduler::internal {
namespace kotlin::gcScheduler {
namespace internal {
class GCSchedulerDataAggressive;
}
class GCScheduler::ThreadData::Impl : private Pinned {
public:
Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept;
internal::GCSchedulerDataAggressive& scheduler() noexcept { return scheduler_; }
internal::MutatorAssists::ThreadData& mutatorAssists() noexcept { return mutatorAssists_; }
private:
internal::GCSchedulerDataAggressive& scheduler_;
internal::MutatorAssists::ThreadData mutatorAssists_;
};
namespace internal {
// The slowpath will trigger GC if this thread didn't meet this safepoint/allocation site before.
class GCSchedulerDataAggressive : public GCSchedulerData {
class GCSchedulerDataAggressive {
public:
GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function<int64_t()> scheduleGC) noexcept :
scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) {
RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized");
}
void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); }
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
void SetAllocatedBytes(size_t bytes) noexcept override {
void setAllocatedBytes(size_t bytes) noexcept {
// Still checking allocations: with a long running loop all safepoints
// might be "met", so that's the only trigger to not run out of memory.
if (heapGrowthController_.SetAllocatedBytes(bytes)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
scheduleGC_();
} else {
safePoint();
auto boundary = heapGrowthController_.boundaryForHeapSize(bytes);
switch (boundary) {
case HeapGrowthController::MemoryBoundary::kNone:
safePoint();
return;
case HeapGrowthController::MemoryBoundary::kTrigger:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
schedule();
return;
case HeapGrowthController::MemoryBoundary::kTarget:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
auto epoch = schedule();
RuntimeLogWarning({kTagGC}, "Pausing the mutators");
mutatorAssists_.requestAssists(epoch);
return;
}
}
void safePoint() noexcept {
if (safePointTracker_.registerCurrentSafePoint(1)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint");
scheduleGC_();
schedule();
}
}
void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {
heapGrowthController_.updateBoundaries(aliveBytes);
// Must wait for all mutators to be released. GC thread cannot continue.
// This is the contract between GC and mutators. With regular native state
// each mutator must check that GC is not doing something. Here GC must check
// that each mutator has done all it needs.
mutatorAssists_.completeEpoch(epoch, [](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& {
return threadData.gcScheduler().impl().mutatorAssists();
});
}
int64_t schedule() noexcept { return scheduleGC_(); }
MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; }
private:
std::function<void()> scheduleGC_;
std::function<int64_t()> scheduleGC_;
HeapGrowthController heapGrowthController_;
SafePointTracker<> safePointTracker_;
mm::SafePointActivator safePointActivator_;
MutatorAssists mutatorAssists_;
};
} // namespace kotlin::gcScheduler::internal
} // namespace internal
class GCScheduler::Impl : private Pinned {
public:
explicit Impl(GCSchedulerConfig& config) noexcept;
internal::GCSchedulerDataAggressive& impl() noexcept { return impl_; }
private:
internal::GCSchedulerDataAggressive impl_;
};
} // namespace kotlin::gcScheduler
kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp
+20 -6
View File
@@ -27,18 +27,18 @@ using namespace kotlin;
TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) {
SKIP_ON_WINDOWS();
[]() OPTNONE {
testing::MockFunction<void()> scheduleGC;
testing::MockFunction<int64_t()> scheduleGC;
gcScheduler::GCSchedulerConfig config;
gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction());
EXPECT_CALL(scheduleGC, Call()).Times(1);
EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(0));
for (int i = 0; i < 10; i++) {
scheduler.safePoint();
}
testing::Mock::VerifyAndClearExpectations(&scheduleGC);
EXPECT_CALL(scheduleGC, Call()).Times(1);
EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1));
scheduler.safePoint();
testing::Mock::VerifyAndClearExpectations(&scheduleGC);
}();
@@ -47,20 +47,34 @@ TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) {
TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) {
SKIP_ON_WINDOWS();
[]() OPTNONE {
testing::MockFunction<void()> scheduleGC;
testing::MockFunction<int64_t()> scheduleGC;
gcScheduler::GCSchedulerConfig config;
config.autoTune = false;
config.targetHeapBytes = 10;
config.heapTriggerCoefficient = 0.9;
gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction());
int i = 0;
// We trigger GC on the first iteration, when the unique allocation point is faced,
// on the second to last iteration when weak target heap size is reached,
// and on the last iteration when target heap size is reached.
EXPECT_CALL(scheduleGC, Call()).WillOnce([&i]() { EXPECT_THAT(i, 0); }).WillOnce([&i]() { EXPECT_THAT(i, 9); });
EXPECT_CALL(scheduleGC, Call())
.WillOnce([&i]() {
EXPECT_THAT(i, 0);
return 0;
})
.WillOnce([&i]() {
EXPECT_THAT(i, 8);
return 1;
})
.WillOnce([&i]() {
EXPECT_THAT(i, 9);
return 2;
});
for (; i < 10; i++) {
scheduler.SetAllocatedBytes(i + 1);
scheduler.setAllocatedBytes(i + 1);
}
testing::Mock::VerifyAndClearExpectations(&scheduleGC);
}();
kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp
+41 -27
View File
@@ -14,45 +14,59 @@
#include "Utils.hpp"
#include "std_support/Memory.hpp"
namespace kotlin::gcScheduler {
namespace test_support {
class GCSchedulerThreadDataTestApi;
namespace kotlin::mm {
class ThreadData;
}
class GCSchedulerThreadData;
class GCSchedulerData {
public:
virtual ~GCSchedulerData() = default;
// The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC`
// when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished.
// TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead.
// Always called by the GC thread.
virtual void OnPerformFullGC() noexcept = 0;
// Always called by the GC thread.
virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0;
// Called by different mutator threads.
virtual void SetAllocatedBytes(size_t bytes) noexcept = 0;
};
namespace kotlin::gcScheduler {
class GCScheduler : private Pinned {
public:
class Impl;
class ThreadData : private Pinned {
public:
class Impl;
ThreadData(GCScheduler&, mm::ThreadData&) noexcept;
~ThreadData();
Impl& impl() noexcept { return *impl_; }
void safePoint() noexcept;
private:
std_support::unique_ptr<Impl> impl_;
};
GCScheduler() noexcept;
~GCScheduler();
Impl& impl() noexcept { return *impl_; }
GCSchedulerConfig& config() noexcept { return config_; }
GCSchedulerData& gcData() noexcept { return *gcData_; }
// Should be called on encountering a safepoint.
void safePoint() noexcept;
// Called by different mutator threads.
void setAllocatedBytes(size_t bytes) noexcept;
// Can be called by any thread.
void schedule() noexcept;
// Can be called by any thread.
void scheduleAndWaitFinished() noexcept;
// Can be called by any thread.
void scheduleAndWaitFinalized() noexcept;
// Always called by the GC thread.
void onGCStart() noexcept;
// Called by the GC thread only.
void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept;
private:
GCSchedulerConfig config_;
std_support::unique_ptr<GCSchedulerData> gcData_;
std_support::unique_ptr<Impl> impl_;
};
} // namespace kotlin::gcScheduler
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp
+47 -6
View File
@@ -13,22 +13,63 @@ namespace kotlin::gcScheduler {
// NOTE: When changing default values, reflect them in GC.kt as well.
struct GCSchedulerConfig {
enum class MutatorAssists {
kDefault,
kEnable,
kDisable,
};
std::atomic<bool> autoTune = true;
// The target interval between collections when Kotlin code is idle. GC will be triggered
// by timer no sooner than this value and no later than twice this value since the previous collection.
std::atomic<int64_t> regularGcIntervalMicroseconds = 10 * 1000 * 1000;
// How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true.
// GC will try to keep object bytes under this amount. If object bytes have
// become bigger than this value, and `mutatorAssists` are enabled the GC will
// stop the world and wait until current epoch finishes.
// Adapts after each GC epoch when `autoTune = true`.
std::atomic<int64_t> targetHeapBytes = 1024 * 1024;
// The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection
// N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped
// between minHeapBytes and maxHeapBytes.
// The rate at which `targetHeapBytes` changes when `autoTune = true`. Concretely: if after the collection
// `N` object bytes remain in the heap, the next `targetHeapBytes` will be `N / targetHeapUtilization` capped
// between `minHeapBytes` and `maxHeapBytes`.
std::atomic<double> targetHeapUtilization = 0.5;
// The minimum value of targetHeapBytes for autoTune = true
// The minimum value of `targetHeapBytes` for `autoTune = true`
std::atomic<int64_t> minHeapBytes = 1024 * 1024;
// The maximum value of targetHeapBytes for autoTune = true
// The maximum value of `targetHeapBytes` for `autoTune = true`
std::atomic<int64_t> maxHeapBytes = std::numeric_limits<int64_t>::max();
// GC will be triggered when object bytes reach `heapTriggerCoefficient * targetHeapBytes`.
std::atomic<double> heapTriggerCoefficient = 0.9;
// See `mutatorAssists()`.
std::atomic<std::underlying_type_t<MutatorAssists>> mutatorAssistsImpl =
static_cast<std::underlying_type_t<MutatorAssists>>(MutatorAssists::kDefault);
std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); }
// Whether mutators should stop and wait for GC to complete when
// current object heap size is bigger than `targetHeapBytes`.
// By default on, unless `autoTune = false` or `maxHeapBytes` is set.
bool mutatorAssists() const noexcept {
switch (static_cast<MutatorAssists>(mutatorAssistsImpl.load())) {
case MutatorAssists::kDisable:
return false;
case MutatorAssists::kEnable:
return true;
case MutatorAssists::kDefault:
// If after a GC epoch the alive set is more than maximum `targetHeapBytes`, the next GC will be
// scheduled instantly and when the assists are turned on, the mutators would be immediately paused.
// This will look like the program has hanged.
// So, by default, disable assisting if `targetHeapBytes` has a non-infinite limit
// (either `autoTune == false`, so `targetHeapBytes` is fixed; or `maxHeapBytes`
// is lower than infinity).
// TODO: Figure out what to do with OOMs.
return autoTune.load() && maxHeapBytes.load() == std::numeric_limits<int64_t>::max();
}
}
// See `mutatorAssists()`.
void setMutatorAssists(bool assist) noexcept {
mutatorAssistsImpl.store(
static_cast<std::underlying_type_t<MutatorAssists>>(assist ? MutatorAssists::kEnable : MutatorAssists::kDisable));
}
};
} // namespace kotlin::gcScheduler
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfigTest.cpp
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "GCSchedulerConfig.hpp"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
using namespace kotlin;
TEST(GCSchedulerConfigTest, DefaultMutatorAssists) {
gcScheduler::GCSchedulerConfig config;
EXPECT_TRUE(config.mutatorAssists());
config.autoTune = false;
EXPECT_FALSE(config.mutatorAssists());
config.autoTune = true;
ASSERT_TRUE(config.mutatorAssists());
config.maxHeapBytes = 1024 * 1024 * 1024;
EXPECT_FALSE(config.mutatorAssists());
config.maxHeapBytes = std::numeric_limits<int64_t>::max();
EXPECT_TRUE(config.mutatorAssists());
}
TEST(GCSchedulerConfigTest, DisabledMutatorAssists) {
gcScheduler::GCSchedulerConfig config;
config.setMutatorAssists(false);
EXPECT_FALSE(config.mutatorAssists());
config.autoTune = false;
EXPECT_FALSE(config.mutatorAssists());
config.autoTune = true;
ASSERT_FALSE(config.mutatorAssists());
config.maxHeapBytes = 1024 * 1024 * 1024;
EXPECT_FALSE(config.mutatorAssists());
config.maxHeapBytes = std::numeric_limits<int64_t>::max();
EXPECT_FALSE(config.mutatorAssists());
}
TEST(GCSchedulerConfigTest, EnabledMutatorAssists) {
gcScheduler::GCSchedulerConfig config;
config.setMutatorAssists(true);
EXPECT_TRUE(config.mutatorAssists());
config.autoTune = false;
EXPECT_TRUE(config.mutatorAssists());
config.autoTune = true;
ASSERT_TRUE(config.mutatorAssists());
config.maxHeapBytes = 1024 * 1024 * 1024;
EXPECT_TRUE(config.mutatorAssists());
config.maxHeapBytes = std::numeric_limits<int64_t>::max();
EXPECT_TRUE(config.mutatorAssists());
}
kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp
+29 -11
View File
@@ -17,17 +17,35 @@ namespace kotlin::gcScheduler::internal {
class HeapGrowthController {
public:
explicit HeapGrowthController(GCSchedulerConfig& config) noexcept :
config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) {}
enum class MemoryBoundary {
// Memory usage is low.
kNone,
// Memory usage is high, GC should be triggered.
kTrigger,
// Memory usage is critical, GC is running behind the mutators. Mutators should pause.
kTarget,
};
// Called by the mutators.
// Returns true if needs GC.
bool SetAllocatedBytes(size_t totalAllocatedBytes) noexcept { return totalAllocatedBytes >= targetHeapBytes_; }
explicit HeapGrowthController(GCSchedulerConfig& config) noexcept :
config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) {
triggerHeapBytes_ = targetHeapBytes_ * config_.heapTriggerCoefficient.load(std::memory_order_relaxed);
}
// Can be called by any thread.
MemoryBoundary boundaryForHeapSize(size_t totalAllocatedBytes) noexcept {
if (totalAllocatedBytes >= targetHeapBytes_) {
return config_.mutatorAssists() ? MemoryBoundary::kTarget : MemoryBoundary::kTrigger;
} else if (totalAllocatedBytes >= triggerHeapBytes_) {
return MemoryBoundary::kTrigger;
} else {
return MemoryBoundary::kNone;
}
}
// Called by the GC thread.
void UpdateAliveSetBytes(size_t bytes) noexcept {
void updateBoundaries(size_t aliveBytes) noexcept {
if (config_.autoTune.load()) {
double targetHeapBytes = static_cast<double>(bytes) / config_.targetHeapUtilization;
double targetHeapBytes = static_cast<double>(aliveBytes) / config_.targetHeapUtilization;
if (!std::isfinite(targetHeapBytes)) {
// This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything.
return;
@@ -35,6 +53,7 @@ public:
double minHeapBytes = static_cast<double>(config_.minHeapBytes.load(std::memory_order_relaxed));
double maxHeapBytes = static_cast<double>(config_.maxHeapBytes.load(std::memory_order_relaxed));
targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes);
triggerHeapBytes_ = static_cast<size_t>(targetHeapBytes * config_.heapTriggerCoefficient.load(std::memory_order_relaxed));
config_.targetHeapBytes.store(static_cast<int64_t>(targetHeapBytes), std::memory_order_relaxed);
targetHeapBytes_ = static_cast<size_t>(targetHeapBytes);
} else {
@@ -42,14 +61,13 @@ public:
}
}
void OnPerformFullGC() noexcept {
// TODO: Need to protect against mutators that can overrun the GC thread.
targetHeapBytes_ = std::numeric_limits<size_t>::max();
}
size_t targetHeapBytes() const noexcept { return targetHeapBytes_; }
size_t triggerHeapBytes() const noexcept { return triggerHeapBytes_; }
private:
GCSchedulerConfig& config_;
size_t targetHeapBytes_ = 0;
size_t triggerHeapBytes_ = 0;
};
} // namespace kotlin::gcScheduler::internal
kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthControllerTest.cpp
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "HeapGrowthController.hpp"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
using namespace kotlin;
using gcScheduler::internal::HeapGrowthController;
using MemoryBoundary = HeapGrowthController::MemoryBoundary;
TEST(HeapGrowthControllerTest, BoundariesWithAssists) {
gcScheduler::GCSchedulerConfig config;
config.targetHeapBytes = 10;
config.heapTriggerCoefficient = 0.7;
ASSERT_TRUE(config.mutatorAssists());
HeapGrowthController controller(config);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
for (size_t i = 0; i < 12; ++i) {
auto expected = i < 7 ? MemoryBoundary::kNone : i < 10 ? MemoryBoundary::kTrigger : MemoryBoundary::kTarget;
EXPECT_THAT(controller.boundaryForHeapSize(i), expected);
}
}
TEST(HeapGrowthControllerTest, BoundariesWithoutAssists) {
gcScheduler::GCSchedulerConfig config;
config.targetHeapBytes = 10;
config.heapTriggerCoefficient = 0.7;
config.setMutatorAssists(false);
HeapGrowthController controller(config);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
for (size_t i = 0; i < 12; ++i) {
auto expected = i < 7 ? MemoryBoundary::kNone : MemoryBoundary::kTrigger;
EXPECT_THAT(controller.boundaryForHeapSize(i), expected);
}
}
TEST(HeapGrowthControllerTest, NoTune) {
gcScheduler::GCSchedulerConfig config;
config.autoTune = false;
config.targetHeapBytes = 10;
config.heapTriggerCoefficient = 0.7;
HeapGrowthController controller(config);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
controller.updateBoundaries(0);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
controller.updateBoundaries(10);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
controller.updateBoundaries(10000);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
controller.updateBoundaries(std::numeric_limits<uint32_t>::max());
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
}
TEST(HeapGrowthControllerTest, Tune) {
gcScheduler::GCSchedulerConfig config;
config.autoTune = true;
config.minHeapBytes = 10;
config.maxHeapBytes = 1000;
config.targetHeapBytes = 100;
config.heapTriggerCoefficient = 0.7;
config.targetHeapUtilization = 0.5;
HeapGrowthController controller(config);
EXPECT_THAT(controller.targetHeapBytes(), 100);
EXPECT_THAT(controller.triggerHeapBytes(), 70);
controller.updateBoundaries(0);
EXPECT_THAT(controller.targetHeapBytes(), 10);
EXPECT_THAT(controller.triggerHeapBytes(), 7);
controller.updateBoundaries(10);
EXPECT_THAT(controller.targetHeapBytes(), 20);
EXPECT_THAT(controller.triggerHeapBytes(), 14);
controller.updateBoundaries(100);
EXPECT_THAT(controller.targetHeapBytes(), 200);
EXPECT_THAT(controller.triggerHeapBytes(), 140);
controller.updateBoundaries(10000);
EXPECT_THAT(controller.targetHeapBytes(), 1000);
EXPECT_THAT(controller.triggerHeapBytes(), 700);
controller.updateBoundaries(std::numeric_limits<uint32_t>::max());
EXPECT_THAT(controller.targetHeapBytes(), 1000);
EXPECT_THAT(controller.triggerHeapBytes(), 700);
}
kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "MutatorAssists.hpp"
#include "CallsChecker.hpp"
#include "KAssert.h"
#include "Logging.hpp"
#include "ThreadData.hpp"
using namespace kotlin;
void gcScheduler::internal::MutatorAssists::ThreadData::safePoint() noexcept {
Epoch epoch = owner_.assistsEpoch_.load(std::memory_order_acquire);
auto noNeedToWait = [this, epoch] { return owner_.completedEpoch_.load(std::memory_order_acquire) >= epoch; };
if (noNeedToWait()) return;
auto prevState = thread_.suspensionData().setStateNoSafePoint(ThreadState::kNative);
RuntimeAssert(prevState == ThreadState::kRunnable, "Expected runnable state");
startedWaiting_.store(epoch * 2, std::memory_order_release);
{
std::unique_lock guard(owner_.m_);
RuntimeLogDebug({kTagGC}, "Thread is assisting for epoch %" PRId64, epoch);
owner_.cv_.wait(guard, noNeedToWait);
RuntimeLogDebug({kTagGC}, "Thread has assisted for epoch %" PRId64, epoch);
}
startedWaiting_.store(epoch * 2 + 1, std::memory_order_release);
// Not doing a safe point. We're a safe point.
prevState = thread_.suspensionData().setStateNoSafePoint(ThreadState::kRunnable);
RuntimeAssert(prevState == ThreadState::kNative, "Expected native state");
}
bool gcScheduler::internal::MutatorAssists::ThreadData::completedEpoch(Epoch epoch) const noexcept {
auto [waitingEpoch, isWaiting] = startedWaiting(std::memory_order_acquire);
if (waitingEpoch > epoch)
// Waiting for an epoch bigger than `epoch` => `epoch` is done here.
return true;
return !isWaiting;
}
void gcScheduler::internal::MutatorAssists::requestAssists(Epoch epoch) noexcept {
RuntimeLogDebug({kTagGC}, "Requesting assists for epoch %" PRId64, epoch);
CallsCheckerIgnoreGuard guard;
std::unique_lock lockGuard(m_);
if (assistsEpoch_.load(std::memory_order_relaxed) >= epoch) {
return;
}
assistsEpoch_.store(epoch, std::memory_order_release);
if (completedEpoch_.load(std::memory_order_relaxed) >= epoch) {
return;
}
RuntimeLogDebug({kTagGC}, "Enabling assists for epoch %" PRId64, epoch);
if (!safePointActivator_) {
safePointActivator_ = mm::SafePointActivator();
}
}
void gcScheduler::internal::MutatorAssists::markEpochCompleted(Epoch epoch) noexcept {
RuntimeLogDebug({kTagGC}, "Disabling assists for epoch %" PRId64, epoch);
{
std::unique_lock guard(m_);
auto previousEpoch = completedEpoch_.exchange(epoch, std::memory_order_release);
RuntimeAssert(
previousEpoch == epoch - 1, "Epochs must be increasing by 1. Previous: %" PRId64 ". Setting: %" PRId64, previousEpoch,
epoch);
if (epoch >= assistsEpoch_.load(std::memory_order_relaxed)) {
safePointActivator_ = std::nullopt;
}
}
cv_.notify_all();
}
kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.hpp
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <mutex>
#include <optional>
#include "SafePoint.hpp"
#include "ThreadRegistry.hpp"
#include "Utils.hpp"
#if KONAN_WINDOWS
#include "ConditionVariable.hpp"
#else
#include <condition_variable>
#endif
namespace kotlin::gcScheduler::internal {
/**
* Coordinating mutator assistance to the GC.
*
* Currently assisting only by pausing threads. So assisting means not creating
* more work for the GC thread.
*
* Threads (both mutators and any other) can call `requestAssists(epoch)` for any
* `epoch` at any time.
*
* If the current GC epoch is greater than `epoch`, the mutators should ignore
* the request to assist.
*
* Otherwise the mutators must wait in the native state
* until the GC thread calls `completeEpoch(epoch)` for epoch >= `epoch`.
*
* The GC thread shall call `completeEpoch(epoch)` once it is done with the epoch,
* and it shall wait for all mutators assisting `epoch` (or lower) to continue.
*/
class MutatorAssists : private Pinned {
public:
using Epoch = int64_t;
class ThreadData : private Pinned {
public:
ThreadData(MutatorAssists& owner, mm::ThreadData& thread) noexcept : owner_(owner), thread_(thread) {}
void safePoint() noexcept;
std::pair<Epoch, bool> startedWaiting(std::memory_order ordering) const noexcept {
auto value = startedWaiting_.load(ordering);
auto waitingEpoch = value / 2;
bool isWaiting = value % 2 == 0;
return {waitingEpoch, isWaiting};
}
private:
friend class MutatorAssists;
bool completedEpoch(Epoch epoch) const noexcept;
MutatorAssists& owner_;
mm::ThreadData& thread_;
// Contains epoch * 2. The lower bit is 1, if completed waiting.
std::atomic<int64_t> startedWaiting_ = 1;
};
// Request all `kRunnable` mutators to start assisting GC for epoch `epoch`.
// Can be called multiple times per `epoch`, and `epoch` may point to the past.
void requestAssists(Epoch epoch) noexcept;
// Should be called by GC, when it completed epoch `epoch`.
// The call blocks waiting for all assisting mutators to finish assisting `epoch`.
// `f` is a map from `mm::ThreadData&` to `MutatorAssists::ThreadData&`.
// Can only be called once per `epoch`, and `epoch` must be increasing
// by exactly 1 every time.
template <typename F>
void completeEpoch(Epoch epoch, F&& f) noexcept {
markEpochCompleted(epoch);
mm::ThreadRegistry::Instance().waitAllThreads(
[f = std::forward<F>(f), epoch](mm::ThreadData& threadData) noexcept { return f(threadData).completedEpoch(epoch); });
}
Epoch assistsRequested(std::memory_order order) noexcept { return assistsEpoch_.load(order); }
private:
void markEpochCompleted(Epoch epoch) noexcept;
std::atomic<Epoch> assistsEpoch_ = 0;
std::atomic<Epoch> completedEpoch_ = 0;
std::optional<mm::SafePointActivator> safePointActivator_;
std::mutex m_;
#if KONAN_WINDOWS
// winpthreads being weird. Using this implementation of condvar means that assisting mutators will spin for the entire duration of the
// GC. This is fine: reaching assisting state should be rare and this state exists to ward of "memory leaks", and additionally assists
// can be disabled.
ConditionVariableSpin cv_;
#else
std::condition_variable cv_;
#endif
};
} // namespace kotlin::gcScheduler::internal
kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssistsTest.cpp
+430
View File
@@ -0,0 +1,430 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "MutatorAssists.hpp"
#include <shared_mutex>
#include <sstream>
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "SafePoint.hpp"
#include "TestSupport.hpp"
#include "std_support/Map.hpp"
using namespace kotlin;
using gcScheduler::internal::MutatorAssists;
using Epoch = MutatorAssists::Epoch;
class MutatorAssistsTest : public ::testing::Test {
public:
class Mutator {
public:
template <typename F>
Mutator(MutatorAssistsTest& owner, F&& f) noexcept :
owner_(owner), thread_([f = std::forward<F>(f), this]() noexcept {
ScopedMemoryInit memory;
{
std::unique_lock guard(initializedMutex_);
threadData_ = memory.memoryState()->GetThreadData();
assists_.emplace(owner_.assists_, *threadData_);
}
owner_.registerMutator(*this);
initialized_.notify_one();
f(*this);
}) {
std::unique_lock guard(initializedMutex_);
initialized_.wait(guard, [this] { return threadData_ && assists_.has_value(); });
}
~Mutator() {
thread_.join();
owner_.unregisterMutator(*this);
}
mm::ThreadData& threadData() noexcept { return *threadData_; }
MutatorAssists::ThreadData& assists() noexcept { return *assists_; }
private:
friend MutatorAssistsTest;
MutatorAssistsTest& owner_;
std::condition_variable initialized_;
std::mutex initializedMutex_;
mm::ThreadData* threadData_;
std::optional<MutatorAssists::ThreadData> assists_;
ScopedThread thread_;
};
void requestAssists(Epoch epoch) noexcept { assists_.requestAssists(epoch); }
void completeEpoch(Epoch epoch) noexcept {
assists_.completeEpoch(epoch, [this](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& {
return getMutator(threadData).assists();
});
}
void safePoint() noexcept {
if (!mm::test_support::safePointsAreActive()) return;
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
getMutator(*threadData).assists().safePoint();
}
private:
void registerMutator(Mutator& mutator) noexcept {
std::unique_lock guard(mutatorMapMutex_);
auto [_, inserted] = mutatorMap_.insert(std::make_pair(&mutator.threadData(), &mutator));
RuntimeAssert(inserted, "Mutator was already inserted");
}
void unregisterMutator(Mutator& mutator) noexcept {
std::unique_lock guard(mutatorMapMutex_);
auto count = mutatorMap_.erase(&mutator.threadData());
RuntimeAssert(count == 1, "Mutator must be in the map");
}
Mutator& getMutator(mm::ThreadData& threadData) noexcept {
std::shared_lock guard(mutatorMapMutex_);
auto it = mutatorMap_.find(&threadData);
RuntimeAssert(it != mutatorMap_.end(), "Mutator must be in the map");
return *it->second;
}
MutatorAssists assists_;
RWSpinLock<MutexThreadStateHandling::kIgnore> mutatorMapMutex_;
std_support::map<mm::ThreadData*, Mutator*> mutatorMap_;
};
TEST_F(MutatorAssistsTest, EnableSafePointsWhenRequestingAssists) {
ASSERT_FALSE(mm::test_support::safePointsAreActive());
requestAssists(1);
EXPECT_TRUE(mm::test_support::safePointsAreActive());
completeEpoch(1);
EXPECT_FALSE(mm::test_support::safePointsAreActive());
}
TEST_F(MutatorAssistsTest, EnableSafePointsWithNestedRequest) {
ASSERT_FALSE(mm::test_support::safePointsAreActive());
requestAssists(1);
ASSERT_TRUE(mm::test_support::safePointsAreActive());
requestAssists(2);
EXPECT_TRUE(mm::test_support::safePointsAreActive());
completeEpoch(1);
EXPECT_TRUE(mm::test_support::safePointsAreActive());
completeEpoch(2);
EXPECT_FALSE(mm::test_support::safePointsAreActive());
}
TEST_F(MutatorAssistsTest, StressEnableSafePointsByMutators) {
constexpr Epoch epochsCount = 4;
std::array<std::atomic<bool>, epochsCount> enabled = {false};
std::atomic<bool> canStart = false;
std::atomic<bool> canStop = false;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&, i](Mutator&) noexcept {
while (!canStart.load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
requestAssists((i % epochsCount) + 1);
enabled[i % epochsCount].store(true, std::memory_order_relaxed);
while (!canStop.load(std::memory_order_relaxed)) {
safePoint();
}
}));
}
ASSERT_FALSE(mm::test_support::safePointsAreActive());
canStart.store(true, std::memory_order_relaxed);
for (Epoch i = 0; i < epochsCount; ++i) {
while (!enabled[i].load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
EXPECT_TRUE(mm::test_support::safePointsAreActive());
completeEpoch(i + 1);
}
EXPECT_FALSE(mm::test_support::safePointsAreActive());
canStop.store(true, std::memory_order_relaxed);
}
TEST_F(MutatorAssistsTest, Assist) {
constexpr Epoch epochsCount = 4;
std::array<std::atomic<bool>, epochsCount> canStart = {false};
std::array<std::atomic<size_t>, epochsCount> started = {0};
std::array<std::atomic<size_t>, epochsCount> finished = {0};
std::atomic<Epoch> gcCompleted = 0;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&](Mutator&) noexcept {
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
while (!canStart[epoch].load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
started[epoch].fetch_add(1, std::memory_order_relaxed);
safePoint();
EXPECT_THAT(gcCompleted.load(std::memory_order_relaxed), epoch);
finished[epoch].fetch_add(1, std::memory_order_relaxed);
}
}));
}
for (auto& m : mutators) {
EXPECT_THAT(m->threadData().state(), ThreadState::kRunnable);
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
requestAssists(epoch + 1);
canStart[epoch].store(true, std::memory_order_relaxed);
while (started[epoch].load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
while (!std::all_of(mutators.begin(), mutators.end(), [epoch](auto& m) noexcept {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
return waitingEpoch == epoch + 1 && waiting;
})) {
std::this_thread::yield();
}
gcCompleted.store(epoch, std::memory_order_relaxed);
for (auto& m : mutators) {
EXPECT_THAT(m->threadData().state(), ThreadState::kNative);
// And already checked that all of them have started waiting for epoch.
}
completeEpoch(epoch + 1);
while (finished[epoch].load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
if (epoch != epochsCount - 1) {
EXPECT_THAT(m->threadData().state(), ThreadState::kRunnable);
}
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, epoch + 1);
EXPECT_FALSE(waiting);
}
}
}
TEST_F(MutatorAssistsTest, AssistNoSync) {
constexpr Epoch epochsCount = 10000;
std::atomic<bool> canStop = false;
std::atomic<size_t> finished = 0;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&](Mutator&) noexcept {
while (!canStop.load(std::memory_order_relaxed)) {
safePoint();
std::this_thread::yield();
}
finished.fetch_add(1, std::memory_order_relaxed);
}));
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
requestAssists(epoch + 1);
completeEpoch(epoch + 1);
}
canStop.store(true, std::memory_order_relaxed);
while (finished.load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, testing::Le(epochsCount));
EXPECT_FALSE(waiting);
}
}
TEST_F(MutatorAssistsTest, AssistWithNativeMutators) {
constexpr Epoch epochsCount = 10000;
std::atomic<bool> canStop = false;
std::atomic<size_t> finished = 0;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&, i](Mutator&) noexcept {
if (i % 2 == 0) {
ThreadStateGuard guard(ThreadState::kNative);
while (!canStop.load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
} else {
while (!canStop.load(std::memory_order_relaxed)) {
safePoint();
std::this_thread::yield();
}
}
finished.fetch_add(1, std::memory_order_relaxed);
}));
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
requestAssists(epoch + 1);
completeEpoch(epoch + 1);
}
canStop.store(true, std::memory_order_relaxed);
while (finished.load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, testing::Le(epochsCount));
EXPECT_FALSE(waiting);
}
}
TEST_F(MutatorAssistsTest, AssistNoRequests) {
constexpr Epoch epochsCount = 10000;
std::atomic<bool> canStart = false;
std::atomic<bool> canStop = false;
std::atomic<size_t> started = 0;
std::atomic<size_t> finished = 0;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&](Mutator&) noexcept {
while (!canStart.load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
started.fetch_add(1, std::memory_order_relaxed);
while (!canStop.load(std::memory_order_relaxed)) {
safePoint();
std::this_thread::yield();
}
finished.fetch_add(1, std::memory_order_relaxed);
}));
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
canStart.store(true, std::memory_order_relaxed);
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
completeEpoch(epoch + 1);
}
canStop.store(true, std::memory_order_relaxed);
while (finished.load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
}
TEST_F(MutatorAssistsTest, AssistRequestsByMutators) {
constexpr Epoch epochsCount = 100;
std::atomic<bool> canStart = false;
std::atomic<bool> canStop = false;
std::atomic<size_t> started = 0;
std::atomic<size_t> finished = 0;
std::atomic<Epoch> currentEpoch = 0;
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&, i](Mutator&) noexcept {
while (!canStart.load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
started.fetch_add(1, std::memory_order_relaxed);
while (!canStop.load(std::memory_order_relaxed)) {
if (i % 2 != 0) {
auto epoch = currentEpoch.load(std::memory_order_relaxed);
requestAssists(epoch + 1);
}
safePoint();
std::this_thread::yield();
}
finished.fetch_add(1, std::memory_order_relaxed);
}));
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
canStart.store(true, std::memory_order_relaxed);
for (Epoch epoch = 0; epoch < epochsCount; ++epoch) {
currentEpoch.store(epoch, std::memory_order_relaxed);
completeEpoch(epoch + 1);
}
canStop.store(true, std::memory_order_relaxed);
while (finished.load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, testing::Le(epochsCount));
EXPECT_FALSE(waiting);
}
}
TEST_F(MutatorAssistsTest, AssistRequestsByMutatorsIntoTheFuture) {
constexpr Epoch epochsCount = 100;
std::atomic<bool> canStart = false;
std::atomic<bool> canStop = false;
std::atomic<size_t> started = 0;
std::atomic<size_t> finished = 0;
std::mutex mutexEpoch;
Epoch scheduledEpoch = 0;
Epoch currentEpoch = 0;
auto scheduleGC = [&]() noexcept -> Epoch {
std::unique_lock guard(mutexEpoch);
if (scheduledEpoch > currentEpoch) return scheduledEpoch;
scheduledEpoch = currentEpoch + 1;
return scheduledEpoch;
};
std_support::vector<std_support::unique_ptr<Mutator>> mutators;
for (int i = 0; i < kDefaultThreadCount; ++i) {
mutators.emplace_back(std_support::make_unique<Mutator>(*this, [&, i](Mutator&) noexcept {
while (!canStart.load(std::memory_order_relaxed)) {
std::this_thread::yield();
}
started.fetch_add(1, std::memory_order_relaxed);
while (!canStop.load(std::memory_order_relaxed)) {
if (i % 2 != 0) {
auto epoch = scheduleGC();
requestAssists(epoch);
}
safePoint();
std::this_thread::yield();
}
finished.fetch_add(1, std::memory_order_relaxed);
}));
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, 0);
EXPECT_FALSE(waiting);
}
canStart.store(true, std::memory_order_relaxed);
for (Epoch epoch = 1; epoch <= epochsCount; ++epoch) {
{
std::unique_lock guard(mutexEpoch);
currentEpoch = epoch;
EXPECT_THAT(currentEpoch, testing::Ge(scheduledEpoch));
}
completeEpoch(epoch);
}
canStop.store(true, std::memory_order_relaxed);
completeEpoch(epochsCount + 1); // The last GC.
while (finished.load(std::memory_order_relaxed) < mutators.size()) {
std::this_thread::yield();
}
for (auto& m : mutators) {
auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed);
EXPECT_THAT(waitingEpoch, testing::Le(epochsCount + 1));
EXPECT_FALSE(waiting);
}
}
kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp
+40 -2
View File
@@ -5,8 +5,46 @@
#include "GCSchedulerImpl.hpp"
#include "CallsChecker.hpp"
#include "GC.hpp"
#include "GlobalData.hpp"
using namespace kotlin;
gcScheduler::GCScheduler::GCScheduler() noexcept : gcData_(std_support::make_unique<internal::GCSchedulerDataManual>()) {}
gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler&, mm::ThreadData&) noexcept :
impl_(std_support::make_unique<Impl>()) {}
ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept {}
gcScheduler::GCScheduler::ThreadData::~ThreadData() = default;
gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique<Impl>()) {}
gcScheduler::GCScheduler::~GCScheduler() = default;
ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept {}
void gcScheduler::GCScheduler::schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
mm::GlobalData::Instance().gc().Schedule();
}
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
CallsCheckerIgnoreGuard guard;
auto& gc = mm::GlobalData::Instance().gc();
auto epoch = gc.Schedule();
NativeOrUnregisteredThreadGuard stateGuard(/* reentrant = */ true);
gc.WaitFinished(epoch);
}
void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
CallsCheckerIgnoreGuard guard;
auto& gc = mm::GlobalData::Instance().gc();
auto epoch = gc.Schedule();
NativeOrUnregisteredThreadGuard stateGuard(/* reentrant = */ true);
gc.WaitFinalizers(epoch);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {}
kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp
+6 -8
View File
@@ -9,15 +9,13 @@
#include "Logging.hpp"
namespace kotlin::gcScheduler::internal {
namespace kotlin::gcScheduler {
class GCSchedulerDataManual : public GCSchedulerData {
class GCScheduler::Impl : private Pinned {
public:
GCSchedulerDataManual() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); }
void OnPerformFullGC() noexcept override {}
void UpdateAliveSetBytes(size_t bytes) noexcept override {}
void SetAllocatedBytes(size_t bytes) noexcept override {}
Impl() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); }
};
} // namespace kotlin::gcScheduler::internal
class GCScheduler::ThreadData::Impl : private Pinned {};
} // namespace kotlin::gcScheduler
kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt
+39 -3
View File
@@ -177,7 +177,6 @@ public object GC {
* When Kotlin code is not allocating enough to trigger GC, the GC scheduler uses timer to drive collection.
* Timer-triggered collection will happen roughly in [regularGCInterval] .. 2 * [regularGCInterval] since
* any previous collection.
* Unused with on-safepoints GC scheduler.
*
* Default: 10 seconds
*
@@ -193,8 +192,9 @@ public object GC {
}
/**
* Total amount of heap available for Kotlin objects. When Kotlin objects overflow this heap,
* the garbage collection is requested. Automatically adjusts when [autotune] is true:
* Total amount of heap available for Kotlin objects. The GC tries to schedule execution
* so that Kotlin heap doesn't overflow this heap.
* Automatically adjusts when [autotune] is true:
* after each collection the [targetHeapBytes] is set to heapBytes / [targetHeapUtilization] and
* capped between [minHeapBytes] and [maxHeapBytes], where heapBytes is heap usage after the garbage
* is collected.
@@ -265,6 +265,30 @@ public object GC {
setMaxHeapBytes(value)
}
/**
* The GC is scheduled when Kotlin heap overflows [heapTriggerCoefficient] * [targetHeapBytes].
*
* Default: 0.9
*
* @throws [IllegalArgumentException] when value is outside (0, 1] interval.
*/
var heapTriggerCoefficient: Double
get() = getHeapTriggerCoefficient()
set(value) {
require(value > 0 && value <= 1) { "heapTriggerCoefficient must be in (0, 1] interval: $value" }
setHeapTriggerCoefficient(value)
}
/**
* If true, the GC will pause Kotlin threads when Kotlin heap overflows [targetHeapBytes]
* and will resume them only after current GC is done.
*
* Default: true, unless [autotune] is false or [maxHeapBytes] is less than [Long.MAX_VALUE].
*/
var pauseOnTargetHeapOverflow: Boolean
get() = getPauseOnTargetHeapOverflow()
set(value) = setPauseOnTargetHeapOverflow(value)
/**
* Deprecated and unused. Always returns null.
*
@@ -356,4 +380,16 @@ public object GC {
@GCUnsafeCall("Kotlin_native_internal_GC_setMaxHeapBytes")
private external fun setMaxHeapBytes(value: Long)
@GCUnsafeCall("Kotlin_native_internal_GC_getHeapTriggerCoefficient")
private external fun getHeapTriggerCoefficient(): Double
@GCUnsafeCall("Kotlin_native_internal_GC_setHeapTriggerCoefficient")
private external fun setHeapTriggerCoefficient(value: Double)
@GCUnsafeCall("Kotlin_native_internal_GC_getPauseOnTargetHeapOverflow")
private external fun getPauseOnTargetHeapOverflow(): Boolean
@GCUnsafeCall("Kotlin_native_internal_GC_setPauseOnTargetHeapOverflow")
private external fun setPauseOnTargetHeapOverflow(value: Boolean)
}
kotlin-native/runtime/src/mm/cpp/Memory.cpp
+22 -11
View File
@@ -105,7 +105,7 @@ extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) {
auto* node = mm::FromMemoryState(state);
if (destroyRuntime) {
ThreadStateGuard guard(state, ThreadState::kRunnable);
node->Get()->gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
// TODO: Why not just destruct `GC` object and its thread data counterpart entirely?
mm::GlobalData::Instance().gc().StopFinalizerThreadIfRunning();
}
@@ -300,15 +300,11 @@ extern "C" RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) {
}
extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) {
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
AssertThreadState(threadData, ThreadState::kRunnable);
threadData->gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
}
extern "C" void Kotlin_native_internal_GC_schedule(ObjHeader*) {
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
AssertThreadState(threadData, ThreadState::kRunnable);
threadData->gc().Schedule();
mm::GlobalData::Instance().gcScheduler().schedule();
}
extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) {
@@ -421,6 +417,23 @@ extern "C" void Kotlin_native_internal_GC_setMinHeapBytes(ObjHeader*, KLong valu
mm::GlobalData::Instance().gcScheduler().config().minHeapBytes = value;
}
extern "C" KDouble Kotlin_native_internal_GC_getHeapTriggerCoefficient(ObjHeader*) {
return mm::GlobalData::Instance().gcScheduler().config().heapTriggerCoefficient.load();
}
extern "C" void Kotlin_native_internal_GC_setHeapTriggerCoefficient(ObjHeader*, KDouble value) {
RuntimeAssert(value > 0 && value <= 1, "Must be handled by the caller");
mm::GlobalData::Instance().gcScheduler().config().heapTriggerCoefficient = value;
}
extern "C" KBoolean Kotlin_native_internal_GC_getPauseOnTargetHeapOverflow(ObjHeader*) {
return mm::GlobalData::Instance().gcScheduler().config().mutatorAssists();
}
extern "C" void Kotlin_native_internal_GC_setPauseOnTargetHeapOverflow(ObjHeader*, KBoolean value) {
mm::GlobalData::Instance().gcScheduler().config().setMutatorAssists(value);
}
extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) {
// TODO: Remove when legacy MM is gone.
RETURN_OBJ(nullptr);
@@ -453,9 +466,7 @@ extern "C" void Kotlin_Any_share(ObjHeader* thiz) {
}
extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) {
auto* threadData = memory->GetThreadData();
AssertThreadState(threadData, ThreadState::kRunnable);
threadData->gc().ScheduleAndWaitFullGCWithFinalizers();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
}
extern "C" RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
@@ -623,5 +634,5 @@ RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakR
}
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
mm::GlobalData::Instance().gcScheduler().gcData().SetAllocatedBytes(totalAllocatedBytes);
mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
}
kotlin-native/runtime/src/mm/cpp/SafePoint.cpp
+1 -1
View File
@@ -26,7 +26,7 @@ void safePointActionImpl(mm::ThreadData& threadData) noexcept {
RuntimeAssert(!recursion, "Recursive safepoint");
AutoReset guard(&recursion, true);
mm::GlobalData::Instance().gcScheduler().safePoint();
threadData.gcScheduler().safePoint();
threadData.gc().safePoint();
threadData.suspensionData().suspendIfRequested();
}
kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
+4
View File
@@ -32,6 +32,7 @@ public:
threadId_(threadId),
globalsThreadQueue_(GlobalsRegistry::Instance()),
specialRefRegistry_(SpecialRefRegistry::instance()),
gcScheduler_(GlobalData::Instance().gcScheduler(), *this),
gc_(GlobalData::Instance().gc(), *this),
suspensionData_(ThreadState::kNative, *this) {}
@@ -53,6 +54,8 @@ public:
std_support::vector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
gcScheduler::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; }
gc::GC::ThreadData& gc() noexcept { return gc_; }
ThreadSuspensionData& suspensionData() { return suspensionData_; }
@@ -76,6 +79,7 @@ private:
ThreadLocalStorage tls_;
SpecialRefRegistry::ThreadQueue specialRefRegistry_;
ShadowStack shadowStack_;
gcScheduler::GCScheduler::ThreadData gcScheduler_;
gc::GC::ThreadData gc_;
std_support::vector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
ThreadSuspensionData suspensionData_;
kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp
+1
View File
@@ -35,6 +35,7 @@ public:
ThreadState state() noexcept { return state_; }
ThreadState setState(ThreadState newState) noexcept;
ThreadState setStateNoSafePoint(ThreadState newState) noexcept { return state_.exchange(newState, std::memory_order_acq_rel); }
bool suspended() noexcept { return suspended_; }
bool suspendedOrNative() noexcept { return suspended() || state() == kotlin::ThreadState::kNative; }