azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Track memory in big chunks ^KT-57773

Browse Source 
Additionally removed testFixtures{} and test{} in gcScheduler/common runtime module,
because that source set became empty.
This commit is contained in:
Alexander Shabalin
2023-06-21 15:29:20 +02:00
committed by Space Team
parent 0a726d4ebf
commit 220ecc4788
43 changed files with 111 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/build.gradle.kts
-2
View File
@@ -370,8 +370,6 @@ bitcode {
headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
sourceSets {
main {}
testFixtures {}
test {}
}
onlyIf { target.supportsThreads() }
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp
+1 -3
View File
@@ -41,8 +41,7 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce
return AlignUp<uint64_t>(heapArrayHeaderSize + membersSize, kObjectAlignment);
}
CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept :
heap_(heap), gcScheduler_(gcScheduler), nextFitPage_(nullptr), extraObjectPage_(nullptr) {
CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) {
CustomAllocInfo("CustomAllocator::CustomAllocator(heap)");
memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_));
}
@@ -126,7 +125,6 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept {
uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes");
gcScheduler_.OnSafePointAllocation(size);
CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size);
uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell);
uint8_t* ptr;
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp
+1 -3
View File
@@ -11,7 +11,6 @@
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GCScheduler.hpp"
#include "Heap.hpp"
#include "NextFitPage.hpp"
#include "Memory.h"
@@ -21,7 +20,7 @@ namespace kotlin::alloc {
class CustomAllocator {
public:
explicit CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept;
explicit CustomAllocator(Heap& heap) noexcept;
ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
@@ -49,7 +48,6 @@ private:
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount) noexcept;
Heap& heap_;
gcScheduler::GCSchedulerThreadData& gcScheduler_;
NextFitPage* nextFitPage_;
FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1];
ExtraObjectPage* extraObjectPage_;
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp
+7 -21
View File
@@ -8,7 +8,6 @@
#include "CustomAllocConstants.hpp"
#include "CustomAllocator.hpp"
#include "GCScheduler.hpp"
#include "Memory.h"
#include "gtest/gtest.h"
#include "Heap.hpp"
@@ -28,9 +27,7 @@ TEST(CustomAllocTest, SmallAllocNonNull) {
fakeTypes[i] = {.typeInfo_ = &fakeTypes[i], .instanceSize_ = 8 * i, .flags_ = 0};
}
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
CustomAllocator ca(heap);
ObjHeader* obj[N];
for (int i = 1; i < N; ++i) {
TypeInfo* type = fakeTypes + i;
@@ -43,9 +40,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) {
const int N = FIXED_BLOCK_PAGE_CELL_COUNT / FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE;
for (int blocks = MIN_BLOCK_SIZE; blocks < FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blocks) {
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
CustomAllocator ca(heap);
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
uint8_t* first = reinterpret_cast<uint8_t*>(ca.CreateObject(&fakeType));
for (int i = 1; i < N; ++i) {
@@ -58,9 +53,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) {
TEST(CustomAllocTest, FixedBlockPageThreshold) {
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
CustomAllocator ca(heap);
const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10;
const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10;
for (int blocks = FROM; blocks <= TO; ++blocks) {
@@ -71,9 +64,7 @@ TEST(CustomAllocTest, FixedBlockPageThreshold) {
TEST(CustomAllocTest, NextFitPageThreshold) {
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
CustomAllocator ca(heap);
const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10;
const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10;
for (int blocks = FROM; blocks <= TO; ++blocks) {
@@ -85,11 +76,8 @@ TEST(CustomAllocTest, NextFitPageThreshold) {
TEST(CustomAllocTest, TwoAllocatorsDifferentPages) {
for (int blocks = MIN_BLOCK_SIZE; blocks < 2000; ++blocks) {
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData1(config, [](auto&) {});
kotlin::gcScheduler::GCSchedulerThreadData schedulerData2(config, [](auto&) {});
CustomAllocator ca1(heap, schedulerData1);
CustomAllocator ca2(heap, schedulerData2);
CustomAllocator ca1(heap);
CustomAllocator ca2(heap);
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
uint8_t* obj1 = reinterpret_cast<uint8_t*>(ca1.CreateObject(&fakeType));
uint8_t* obj2 = reinterpret_cast<uint8_t*>(ca2.CreateObject(&fakeType));
@@ -102,9 +90,7 @@ using Data = typename kotlin::mm::ExtraObjectData;
TEST(CustomAllocTest, AllocExtraObjectNonNullZeroed) {
Heap heap;
kotlin::gcScheduler::GCSchedulerConfig config;
kotlin::gcScheduler::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
CustomAllocator ca(heap);
for (int i = 1; i < 10; ++i) {
uint8_t* obj = reinterpret_cast<uint8_t*>(ca.CreateExtraObject());
EXPECT_TRUE(obj);
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+2 -1
View File
@@ -98,7 +98,8 @@ void* SafeAlloc(uint64_t size) noexcept {
konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes: %s. Aborting.\n", size, strerror(errno));
konan::abort();
}
allocatedBytesCounter.fetch_add(static_cast<size_t>(size), std::memory_order_relaxed);
auto previousSize = allocatedBytesCounter.fetch_add(static_cast<size_t>(size), std::memory_order_relaxed);
OnMemoryAllocation(previousSize + static_cast<size_t>(size));
CustomAllocDebug("SafeAlloc(%zu) = %p", static_cast<size_t>(size), memory);
return memory;
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+1 -6
View File
@@ -59,10 +59,6 @@ struct ProcessWeaksTraits {
} // namespace
void gc::ConcurrentMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
gcScheduler_.OnSafePointAllocation(size);
}
void gc::ConcurrentMarkAndSweep::ThreadData::Schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
@@ -187,8 +183,6 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
mm::WaitForThreadsSuspension();
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
#ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread
@@ -228,6 +222,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
}
#endif
scheduler.gcData().UpdateAliveSetBytes(allocatedBytes());
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+1 -6
View File
@@ -78,13 +78,9 @@ public:
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
explicit ThreadData(
ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept :
gc_(gc), threadData_(threadData), gcScheduler_(gcScheduler) {}
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
~ThreadData() = default;
void SafePointAllocation(size_t size) noexcept;
void Schedule() noexcept;
void ScheduleAndWaitFullGC() noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept;
@@ -103,7 +99,6 @@ public:
friend ConcurrentMarkAndSweep;
ConcurrentMarkAndSweep& gc_;
mm::ThreadData& threadData_;
gcScheduler::GCSchedulerThreadData& gcScheduler_;
std::atomic<bool> marking_;
BarriersThreadData barriers_;
};
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+1 -2
View File
@@ -15,8 +15,7 @@
using namespace kotlin;
gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
impl_(std_support::make_unique<Impl>(gc, gcScheduler, threadData)) {}
gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique<Impl>(gc, threadData)) {}
gc::GC::ThreadData::~ThreadData() = default;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp
+3 -3
View File
@@ -44,14 +44,14 @@ private:
class GC::ThreadData::Impl : private Pinned {
public:
Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData, gcScheduler),
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData),
#ifndef CUSTOM_ALLOCATOR
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
}
#else
alloc_(gc.impl_->gc().heap(), gcScheduler) {
alloc_(gc.impl_->gc().heap()) {
}
#endif
kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp
-1
View File
@@ -28,7 +28,6 @@ public:
AllocatorWithGC(BaseAllocator base, GCThreadData& gc) noexcept : base_(std::move(base)), gc_(gc) {}
void* Alloc(size_t size) noexcept {
gc_.SafePointAllocation(size);
if (void* ptr = base_.Alloc(size)) {
return ptr;
}
kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp
-4
View File
@@ -33,7 +33,6 @@ private:
class MockGC {
public:
MOCK_METHOD(void, SafePointAllocation, (size_t));
MOCK_METHOD(void, OnOOM, (size_t));
};
@@ -46,7 +45,6 @@ TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
testing::StrictMock<MockGC> gc;
{
testing::InSequence seq;
EXPECT_CALL(gc, SafePointAllocation(size));
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
EXPECT_CALL(gc, OnOOM(_)).Times(0);
}
@@ -62,7 +60,6 @@ TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
testing::StrictMock<MockGC> gc;
{
testing::InSequence seq;
EXPECT_CALL(gc, SafePointAllocation(size));
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
EXPECT_CALL(gc, OnOOM(size));
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
@@ -78,7 +75,6 @@ TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
testing::StrictMock<MockGC> gc;
{
testing::InSequence seq;
EXPECT_CALL(gc, SafePointAllocation(size));
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
EXPECT_CALL(gc, OnOOM(size));
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+1 -1
View File
@@ -29,7 +29,7 @@ public:
public:
class Impl;
ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept;
ThreadData(GC& gc, mm::ThreadData& threadData) noexcept;
~ThreadData();
Impl& impl() noexcept { return *impl_; }
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp
-1
View File
@@ -309,7 +309,6 @@ void GCHandle::marked(kotlin::gc::MarkStats stats) {
if (!stat->markStats) {
stat->markStats = MarkStats{};
}
stat->markStats->markedSizeBytes += stats.markedSizeBytes;
stat->markStats->markedCount += stats.markedCount;
}
}
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp
+1 -5
View File
@@ -32,7 +32,6 @@ struct SweepStats {
struct MarkStats {
uint64_t markedCount = 0;
uint64_t markedSizeBytes = 0;
};
class GCHandle {
@@ -103,10 +102,7 @@ public:
explicit GCMarkScope(GCHandle& handle);
~GCMarkScope();
void addObject(uint64_t objectSize) noexcept {
++stats_.markedCount;
stats_.markedSizeBytes += objectSize;
}
void addObject() noexcept { ++stats_.markedCount; }
};
class GCProcessWeaksScope : GCStageScopeUsTimer, Pinned {
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+2 -5
View File
@@ -81,7 +81,7 @@ void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::
// Do not schedule RegularWeakReferenceImpl but process it right away.
// This will skip markQueue interaction.
if (Traits::tryMark(weakReference)) {
markHandle.addObject(mm::GetAllocatedHeapSize(weakReference));
markHandle.addObject();
// RegularWeakReferenceImpl is empty, but keeping this just in case.
Traits::processInMark(markQueue, weakReference);
}
@@ -94,9 +94,7 @@ template <typename Traits>
void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept {
auto markHandle = handle.mark();
while (ObjHeader* top = Traits::tryDequeue(markQueue)) {
// TODO: Consider moving it to the sweep phase to make this loop more tight.
// This, however, requires care with scheduler interoperation.
markHandle.addObject(mm::GetAllocatedHeapSize(top));
markHandle.addObject();
Traits::processInMark(markQueue, top);
@@ -169,7 +167,6 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T
template <typename Traits>
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
auto handle = gcHandle.collectThreadRoots(thread);
thread.gcScheduler().OnStoppedForGC();
// TODO: Remove useless mm::ThreadRootSet abstraction.
for (auto value : mm::ThreadRootSet(thread)) {
if (internal::collectRoot<Traits>(markQueue, value.object)) {
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp
-9
View File
@@ -157,19 +157,10 @@ private:
ScopedMarkTraits markTraits_;
};
size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<test_support::Any>> objects) {
size_t size = 0;
for (auto& object : objects) {
size += mm::GetAllocatedHeapSize(object.get().header());
}
return size;
}
#define EXPECT_MARKED(stats, ...) \
do { \
std::initializer_list<std::reference_wrapper<test_support::Any>> objects = {__VA_ARGS__}; \
EXPECT_THAT(stats.markedCount, objects.size()); \
EXPECT_THAT(stats.markedSizeBytes, GetObjectsSize(objects)); \
EXPECT_THAT(marked(), MarkedMatcher(objects)); \
} while (false)
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+1 -6
View File
@@ -15,12 +15,7 @@
using namespace kotlin;
gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
#ifdef CUSTOM_ALLOCATOR
impl_(std_support::make_unique<Impl>(gc, gcScheduler, threadData)) {}
#else
impl_(std_support::make_unique<Impl>(gc, threadData)) {}
#endif
gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique<Impl>(gc, threadData)) {}
gc::GC::ThreadData::~ThreadData() = default;
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp
+1 -2
View File
@@ -42,8 +42,7 @@ private:
class GC::ThreadData::Impl : private Pinned {
public:
#ifdef CUSTOM_ALLOCATOR
Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
alloc_(gc.impl_->gc().heap(), gcScheduler) {}
Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {}
#else
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp
-2
View File
@@ -39,8 +39,6 @@ public:
ThreadData() noexcept {}
~ThreadData() = default;
void SafePointAllocation(size_t size) noexcept {}
void Schedule() noexcept {}
void ScheduleAndWaitFullGC() noexcept {}
void ScheduleAndWaitFullGCWithFinalizers() noexcept {}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+1 -2
View File
@@ -15,8 +15,7 @@
using namespace kotlin;
gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
impl_(std_support::make_unique<Impl>(gc, gcScheduler, threadData)) {}
gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique<Impl>(gc, threadData)) {}
gc::GC::ThreadData::~ThreadData() = default;
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp
+4 -3
View File
@@ -42,10 +42,11 @@ private:
class GC::ThreadData::Impl : private Pinned {
public:
Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData, gcScheduler),
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData),
#ifdef CUSTOM_ALLOCATOR
alloc_(gc.impl_->gc().heap(), gcScheduler) {}
alloc_(gc.impl_->gc().heap()) {
}
#else
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+2 -6
View File
@@ -58,10 +58,6 @@ struct ProcessWeaksTraits {
} // namespace
void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
gcScheduler_.OnSafePointAllocation(size);
}
void gc::SameThreadMarkAndSweep::ThreadData::Schedule() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
ThreadStateGuard guard(ThreadState::kNative);
@@ -161,8 +157,6 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [](mm::ThreadData&) { return true; });
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
@@ -181,6 +175,8 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
auto finalizerQueue = heap_.Sweep(gcHandle);
#endif
scheduler.gcData().UpdateAliveSetBytes(allocatedBytes());
mm::ResumeThreads();
gcHandle.threadsAreResumed();
state_.finish(epoch);
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+1 -5
View File
@@ -79,12 +79,9 @@ public:
using ObjectData = SameThreadMarkAndSweep::ObjectData;
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept :
gc_(gc), gcScheduler_(gcScheduler) {}
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc) {}
~ThreadData() = default;
void SafePointAllocation(size_t size) noexcept;
void Schedule() noexcept;
void ScheduleAndWaitFullGC() noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept;
@@ -96,7 +93,6 @@ public:
private:
SameThreadMarkAndSweep& gc_;
gcScheduler::GCSchedulerThreadData& gcScheduler_;
};
using Allocator = ThreadData::Allocator;
kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp
+7 -8
View File
@@ -38,14 +38,6 @@ public:
RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized");
}
void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override {
heapGrowthController_.OnAllocated(threadData.allocatedBytes());
if (heapGrowthController_.NeedsGC()) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
scheduleGC_();
}
}
void OnPerformFullGC() noexcept override {
heapGrowthController_.OnPerformFullGC();
regularIntervalPacer_.OnPerformFullGC();
@@ -54,6 +46,13 @@ public:
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
void SetAllocatedBytes(size_t bytes) noexcept override {
if (heapGrowthController_.SetAllocatedBytes(bytes)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
scheduleGC_();
}
}
private:
GCSchedulerConfig& config_;
std::function<void()> scheduleGC_;
kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp
+15 -16
View File
@@ -10,7 +10,6 @@
#include "AppStateTrackingTestSupport.hpp"
#include "ClockTestSupport.hpp"
#include "GCSchedulerTestSupport.hpp"
#include "SingleThreadExecutor.hpp"
#include "TestSupport.hpp"
#include "std_support/Vector.hpp"
@@ -21,25 +20,18 @@ namespace {
class MutatorThread : private Pinned {
public:
MutatorThread(gcScheduler::GCSchedulerConfig& config, std::function<void(gcScheduler::GCSchedulerThreadData&)> slowPath) :
executor_([&config, slowPath = std::move(slowPath)] { return Context(config, std::move(slowPath)); }) {}
explicit MutatorThread(gcScheduler::GCSchedulerData& scheduler) : executor_([&scheduler] { return Context{scheduler}; }) {}
std::future<void> Allocate(size_t bytes) {
std::future<void> SetAllocatedBytes(size_t bytes) {
return executor_.execute([&, bytes] {
auto& context = executor_.context();
context.threadDataTestApi.SetAllocatedBytes(bytes);
context.slowPath(context.threadData);
context.scheduler.SetAllocatedBytes(bytes);
});
}
private:
struct Context {
gcScheduler::GCSchedulerThreadData threadData;
gcScheduler::test_support::GCSchedulerThreadDataTestApi threadDataTestApi;
std::function<void(gcScheduler::GCSchedulerThreadData&)> slowPath;
Context(gcScheduler::GCSchedulerConfig& config, std::function<void(gcScheduler::GCSchedulerThreadData&)> slowPath) :
threadData(config, [](gcScheduler::GCSchedulerThreadData&) {}), threadDataTestApi(threadData), slowPath(slowPath) {}
gcScheduler::GCSchedulerData& scheduler;
};
SingleThreadExecutor<Context> executor_;
@@ -51,16 +43,22 @@ public:
explicit GCSchedulerDataTestApi(gcScheduler::GCSchedulerConfig& config) : scheduler_(config, scheduleGC_.AsStdFunction()) {
mutators_.reserve(MutatorCount);
for (int i = 0; i < MutatorCount; ++i) {
mutators_.emplace_back(std_support::make_unique<MutatorThread>(
config, [this](gcScheduler::GCSchedulerThreadData& threadData) { scheduler_.UpdateFromThreadData(threadData); }));
mutators_.emplace_back(std_support::make_unique<MutatorThread>(scheduler_));
}
}
std::future<void> Allocate(int mutator, size_t bytes) { return mutators_[mutator]->Allocate(bytes); }
std::future<void> Allocate(int mutator, size_t bytes) {
size_t allocatedBytes = allocatedBytes_.fetch_add(bytes);
allocatedBytes += bytes;
return mutators_[mutator]->SetAllocatedBytes(allocatedBytes);
}
void OnPerformFullGC() { scheduler_.OnPerformFullGC(); }
void UpdateAliveSetBytes(size_t bytes) { scheduler_.UpdateAliveSetBytes(bytes); }
void UpdateAliveSetBytes(size_t bytes) {
allocatedBytes_.store(bytes);
scheduler_.UpdateAliveSetBytes(bytes);
}
testing::MockFunction<void()>& scheduleGC() { return scheduleGC_; }
@@ -70,6 +68,7 @@ public:
}
private:
std::atomic<size_t> allocatedBytes_ = 0;
std_support::vector<std_support::unique_ptr<MutatorThread>> mutators_;
testing::MockFunction<void()> scheduleGC_;
gcScheduler::internal::GCSchedulerDataAdaptive<test_support::manual_clock> scheduler_;
kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp
+6 -10
View File
@@ -22,16 +22,15 @@ class GCSchedulerDataAggressive : public GCSchedulerData {
public:
GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) {
// Trigger the slowpath on each allocation.
config.allocationThresholdBytes = 1;
RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized");
}
void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override {
heapGrowthController_.OnAllocated(threadData.allocatedBytes());
if (heapGrowthController_.NeedsGC()) {
// Still checking allocations: with a long running loop all safepoints
// might be "met", so that's the only trigger to not run out of memory.
void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); }
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
void SetAllocatedBytes(size_t bytes) noexcept override {
// Still checking allocations: with a long running loop all safepoints
// might be "met", so that's the only trigger to not run out of memory.
if (heapGrowthController_.SetAllocatedBytes(bytes)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
scheduleGC_();
} else {
@@ -39,9 +38,6 @@ public:
}
}
void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); }
void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); }
void safePoint() noexcept {
if (safePointTracker_.registerCurrentSafePoint(1)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint");
kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp
+2 -7
View File
@@ -50,14 +50,9 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) {
testing::MockFunction<void()> scheduleGC;
gcScheduler::GCSchedulerConfig config;
gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction());
gcScheduler::GCSchedulerThreadData threadSchedulerData(
config, [&scheduler](gcScheduler::GCSchedulerThreadData& data) { scheduler.UpdateFromThreadData(data); });
ASSERT_EQ(config.allocationThresholdBytes, 1);
config.autoTune = false;
config.targetHeapBytes = 10;
gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction());
int i = 0;
// We trigger GC on the first iteration, when the unique allocation point is faced,
@@ -65,7 +60,7 @@ TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) {
EXPECT_CALL(scheduleGC, Call()).WillOnce([&i]() { EXPECT_THAT(i, 0); }).WillOnce([&i]() { EXPECT_THAT(i, 9); });
for (; i < 10; i++) {
threadSchedulerData.OnSafePointAllocation(1);
scheduler.SetAllocatedBytes(i + 1);
}
testing::Mock::VerifyAndClearExpectations(&scheduleGC);
}();
kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp
+2 -48
View File
@@ -26,9 +26,6 @@ class GCSchedulerData {
public:
virtual ~GCSchedulerData() = default;
// Called by different mutator threads.
virtual void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept = 0;
// The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC`
// when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished.
// TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead.
@@ -38,48 +35,9 @@ public:
// Always called by the GC thread.
virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0;
};
class GCSchedulerThreadData {
public:
explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> slowPath) noexcept :
config_(config), slowPath_(std::move(slowPath)) {
ClearCountersAndUpdateThresholds();
}
// Should be called on encountering a safepoint placed by the allocator.
// TODO: Should this even be a safepoint (i.e. a place, where we suspend)?
void OnSafePointAllocation(size_t size) noexcept {
allocatedBytes_ += size;
if (allocatedBytes_ < allocatedBytesThreshold_) {
return;
}
OnSafePointSlowPath();
}
void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); }
size_t allocatedBytes() const noexcept { return allocatedBytes_; }
private:
friend class test_support::GCSchedulerThreadDataTestApi;
void OnSafePointSlowPath() noexcept {
slowPath_(*this);
ClearCountersAndUpdateThresholds();
}
void ClearCountersAndUpdateThresholds() noexcept {
allocatedBytes_ = 0;
allocatedBytesThreshold_ = config_.allocationThresholdBytes;
}
GCSchedulerConfig& config_;
std::function<void(GCSchedulerThreadData&)> slowPath_;
size_t allocatedBytes_ = 0;
size_t allocatedBytesThreshold_ = 0;
// Called by different mutator threads.
virtual void SetAllocatedBytes(size_t bytes) noexcept = 0;
};
class GCScheduler : private Pinned {
@@ -89,10 +47,6 @@ public:
GCSchedulerConfig& config() noexcept { return config_; }
GCSchedulerData& gcData() noexcept { return *gcData_; }
GCSchedulerThreadData NewThreadData() noexcept {
return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->UpdateFromThreadData(threadData); });
}
// Should be called on encountering a safepoint.
void safePoint() noexcept;
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp
-2
View File
@@ -13,8 +13,6 @@ namespace kotlin::gcScheduler {
// NOTE: When changing default values, reflect them in GC.kt as well.
struct GCSchedulerConfig {
// How many object bytes a thread must allocate to trigger slow path.
std::atomic<int64_t> allocationThresholdBytes = 10 * 1024;
std::atomic<bool> autoTune = true;
// The target interval between collections when Kotlin code is idle. GC will be triggered
// by timer no sooner than this value and no later than twice this value since the previous collection.
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTest.cpp
-132
View File
@@ -1,132 +0,0 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "GCScheduler.hpp"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "GCSchedulerTestSupport.hpp"
using namespace kotlin;
using testing::_;
TEST(GCSchedulerThreadDataTest, AllocationSafePoint) {
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(gcScheduler::GCSchedulerThreadData&)> slowPath;
gcScheduler::GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold);
});
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_CALL(slowPath, Call(_)).Times(0);
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kSize);
}
TEST(GCSchedulerThreadDataTest, ResetByGC) {
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(gcScheduler::GCSchedulerThreadData&)> slowPath;
gcScheduler::GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
EXPECT_CALL(slowPath, Call(_)).Times(0);
scheduler.OnStoppedForGC();
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
}
TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterResetByGC) {
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(gcScheduler::GCSchedulerThreadData&)> slowPath;
gcScheduler::GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
config.allocationThresholdBytes = kAllocationThreshold - kSize;
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
scheduler.OnStoppedForGC();
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
});
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
}
TEST(GCSchedulerThreadDataTest, UpdateThresholdsAfterAllocationSafePoint) {
constexpr size_t kSize = 2;
constexpr size_t kCount = 10;
constexpr size_t kAllocationThreshold = kCount * kSize;
testing::MockFunction<void(gcScheduler::GCSchedulerThreadData&)> slowPath;
gcScheduler::GCSchedulerConfig config;
config.allocationThresholdBytes = kAllocationThreshold;
gcScheduler::GCSchedulerThreadData scheduler(config, slowPath.AsStdFunction());
gcScheduler::test_support::GCSchedulerThreadDataTestApi schedulerTestApi(scheduler);
config.allocationThresholdBytes = kAllocationThreshold - kSize;
EXPECT_CALL(slowPath, Call(_)).Times(0);
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold);
});
scheduler.OnSafePointAllocation(kSize);
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
EXPECT_CALL(slowPath, Call(testing::Ref(scheduler))).WillOnce([&](gcScheduler::GCSchedulerThreadData& scheduler) {
EXPECT_THAT(scheduler.allocatedBytes(), kAllocationThreshold - kSize);
});
for (size_t i = 0; i < kCount - 1; ++i) {
scheduler.OnSafePointAllocation(kSize);
}
testing::Mock::VerifyAndClearExpectations(&slowPath);
EXPECT_THAT(scheduler.allocatedBytes(), 0);
}
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.cpp
-6
View File
@@ -1,6 +0,0 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "GCSchedulerTestSupport.hpp"
kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerTestSupport.hpp
-22
View File
@@ -1,22 +0,0 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include "GCScheduler.hpp"
namespace kotlin::gcScheduler::test_support {
class GCSchedulerThreadDataTestApi : private Pinned {
public:
explicit GCSchedulerThreadDataTestApi(GCSchedulerThreadData& scheduler) : scheduler_(scheduler) {}
void SetAllocatedBytes(size_t bytes) { scheduler_.allocatedBytes_ = bytes; }
private:
GCSchedulerThreadData& scheduler_;
};
} // namespace kotlin::gcScheduler::test_support
kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp
+14 -19
View File
@@ -17,44 +17,39 @@ namespace kotlin::gcScheduler::internal {
class HeapGrowthController {
public:
explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : config_(config) {}
explicit HeapGrowthController(GCSchedulerConfig& config) noexcept :
config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) {}
// Called by the mutators.
void OnAllocated(size_t allocatedBytes) noexcept { allocatedBytes_ += allocatedBytes; }
// Called by the GC thread.
void OnPerformFullGC() noexcept { allocatedBytes_ = 0; }
// Returns true if needs GC.
bool SetAllocatedBytes(size_t totalAllocatedBytes) noexcept { return totalAllocatedBytes >= targetHeapBytes_; }
// Called by the GC thread.
void UpdateAliveSetBytes(size_t bytes) noexcept {
lastAliveSetBytes_ = bytes;
if (config_.autoTune.load()) {
double targetHeapBytes = static_cast<double>(bytes) / config_.targetHeapUtilization;
if (!std::isfinite(targetHeapBytes)) {
// This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything.
return;
}
double minHeapBytes = static_cast<double>(config_.minHeapBytes.load());
double maxHeapBytes = static_cast<double>(config_.maxHeapBytes.load());
double minHeapBytes = static_cast<double>(config_.minHeapBytes.load(std::memory_order_relaxed));
double maxHeapBytes = static_cast<double>(config_.maxHeapBytes.load(std::memory_order_relaxed));
targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes);
config_.targetHeapBytes = static_cast<int64_t>(targetHeapBytes);
config_.targetHeapBytes.store(static_cast<int64_t>(targetHeapBytes), std::memory_order_relaxed);
targetHeapBytes_ = static_cast<size_t>(targetHeapBytes);
} else {
targetHeapBytes_ = config_.targetHeapBytes.load(std::memory_order_relaxed);
}
}
// Called by the mutators.
bool NeedsGC() const noexcept {
uint64_t currentHeapBytes = allocatedBytes_.load() + lastAliveSetBytes_.load();
uint64_t targetHeapBytes = config_.targetHeapBytes;
return currentHeapBytes >= targetHeapBytes;
void OnPerformFullGC() noexcept {
// TODO: Need to protect against mutators that can overrun the GC thread.
targetHeapBytes_ = std::numeric_limits<size_t>::max();
}
private:
GCSchedulerConfig& config_;
// Updated by both the mutators and the GC thread.
std::atomic<size_t> allocatedBytes_ = 0;
// Updated by the GC thread, read by the mutators.
std::atomic<size_t> lastAliveSetBytes_ = 0;
size_t targetHeapBytes_ = 0;
};
} // namespace kotlin::gcScheduler::internal
kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp
+1 -1
View File
@@ -15,9 +15,9 @@ class GCSchedulerDataManual : public GCSchedulerData {
public:
GCSchedulerDataManual() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); }
void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept override {}
void OnPerformFullGC() noexcept override {}
void UpdateAliveSetBytes(size_t bytes) noexcept override {}
void SetAllocatedBytes(size_t bytes) noexcept override {}
};
} // namespace kotlin::gcScheduler::internal
kotlin-native/runtime/src/legacymm/cpp/Memory.cpp
+2
View File
@@ -3786,3 +3786,5 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {}
bool kotlin::FinalizersThreadIsRunning() noexcept {
return false;
}
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {}
kotlin-native/runtime/src/main/cpp/Memory.h
+2
View File
@@ -553,6 +553,8 @@ extern const bool kSupportsMultipleMutators;
void StartFinalizerThreadIfNeeded() noexcept;
bool FinalizersThreadIsRunning() noexcept;
void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept;
} // namespace kotlin
RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object);
kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt
+2 -4
View File
@@ -135,14 +135,12 @@ public object GC {
}
/**
* How many bytes a thread can allocate before informing the GC scheduler.
*
* Default: 10 * 1024
* Deprecated and unused.
*
* Legacy MM: GC allocation threshold, controlling how many bytes allocated since last
* collection will trigger new GC.
*
* Default: (legacy MM) 8 * 1024 * 1024
* Default: 8 * 1024 * 1024
*
* @throws [IllegalArgumentException] when value is not positive.
*/
kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h
+2
View File
@@ -375,6 +375,8 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
#if KONAN_MI_MALLOC
mi_decl_nodiscard mi_decl_export size_t mi_allocated_size(void) mi_attr_noexcept;
// This hook must be implemented by the embedder. `allocated_size` is the new size of the heap.
mi_decl_export void mi_hook_allocation(size_t allocated_size) mi_attr_noexcept;
#endif
#ifdef __cplusplus
kotlin-native/runtime/src/mimalloc/c/region.c
+2 -1
View File
@@ -387,7 +387,8 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l
if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed
#endif
#if KONAN_MI_MALLOC
mi_atomic_add_relaxed(&allocated_size, size);
size_t prev_allocated_size = mi_atomic_add_relaxed(&allocated_size, size);
mi_hook_allocation(prev_allocated_size + size);
#endif
}
return p;
kotlin-native/runtime/src/mm/cpp/Memory.cpp
+9 -3
View File
@@ -358,12 +358,14 @@ extern "C" int64_t Kotlin_native_internal_GC_getCollectCyclesThreshold(ObjHeader
}
extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, int64_t value) {
RuntimeAssert(value > 0, "Must be handled by the caller");
mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = value;
// TODO: Remove when legacy MM is gone.
// Nothing to do
}
extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) {
return mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load();
// TODO: Remove when legacy MM is gone.
// Nothing to do
return 0;
}
extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) {
@@ -619,3 +621,7 @@ RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHea
RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakRef) {
mm::disposeRegularWeakReferenceImpl(weakRef);
}
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
mm::GlobalData::Instance().gcScheduler().gcData().SetAllocatedBytes(totalAllocatedBytes);
}
kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
+1 -6
View File
@@ -11,7 +11,6 @@
#include "GlobalData.hpp"
#include "GlobalsRegistry.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp"
#include "ObjectFactory.hpp"
#include "ShadowStack.hpp"
#include "SpecialRefRegistry.hpp"
@@ -33,8 +32,7 @@ public:
threadId_(threadId),
globalsThreadQueue_(GlobalsRegistry::Instance()),
specialRefRegistry_(SpecialRefRegistry::instance()),
gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()),
gc_(GlobalData::Instance().gc(), gcScheduler_, *this),
gc_(GlobalData::Instance().gc(), *this),
suspensionData_(ThreadState::kNative, *this) {}
~ThreadData() = default;
@@ -55,8 +53,6 @@ public:
std_support::vector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
gcScheduler::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
gc::GC::ThreadData& gc() noexcept { return gc_; }
ThreadSuspensionData& suspensionData() { return suspensionData_; }
@@ -80,7 +76,6 @@ private:
ThreadLocalStorage tls_;
SpecialRefRegistry::ThreadQueue specialRefRegistry_;
ShadowStack shadowStack_;
gcScheduler::GCSchedulerThreadData gcScheduler_;
gc::GC::ThreadData gc_;
std_support::vector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
ThreadSuspensionData suspensionData_;
kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp
+4
View File
@@ -72,3 +72,7 @@ void kotlin::compactObjectPoolInMainThread() noexcept {
size_t kotlin::allocatedBytes() noexcept {
return mi_allocated_size();
}
extern "C" void mi_hook_allocation(size_t allocated_size) mi_attr_noexcept {
OnMemoryAllocation(allocated_size);
}
kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp
+9 -3
View File
@@ -9,6 +9,8 @@
#include <atomic>
#endif
#include "Memory.h"
#if KONAN_INTERNAL_DLMALLOC
extern "C" void* dlcalloc(size_t, size_t);
extern "C" void dlfree(void*);
@@ -37,13 +39,17 @@ size_t allocatedBytesCounter = 0;
void kotlin::initObjectPool() noexcept {}
void* kotlin::allocateInObjectPool(size_t size) noexcept {
// TODO: Check that alignment to kObjectAlignment is satisfied.
void* result = callocImpl(1, size);
#ifndef KONAN_NO_THREADS
allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed);
auto newSize = allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed);
newSize += size;
#else
allocatedBytesCounter += size;
auto newSize = allocatedBytesCounter;
#endif
// TODO: Check that alignment to kObjectAlignment is satisfied.
return callocImpl(1, size);
OnMemoryAllocation(newSize);
return result;
}
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {