azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Add separate FinalizerProcessor for main thread ^KT-63423

Browse Source
This commit is contained in:
Alexander Shabalin
2023-12-01 17:15:48 +01:00
committed by Space Team
parent 8a86fec38f
commit 30d6fa730e
27 changed files with 461 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/alloc/common/cpp/MainThreadFinalizerProcessor.hpp
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <atomic>
#include "RunLoopFinalizerProcessor.hpp"
#include "Utils.hpp"
namespace kotlin::alloc {
// Finalizer processor that runs on the main thread.
//
// Only enabled if `compiler::objcDisposeOnMain()` is true and if the main run loop is processed.
//
// It just wraps `RunLoopFinalizerProcessor`, see it for implementation details.
template <typename FinalizerQueue, typename FinalizerQueueTraits>
class MainThreadFinalizerProcessor : private Pinned {
public:
MainThreadFinalizerProcessor() noexcept {
#if KONAN_HAS_FOUNDATION_FRAMEWORK
if (compiler::objcDisposeOnMain()) {
CFRunLoopPerformBlock(CFRunLoopGetMain(), kCFRunLoopDefaultMode, ^{
[[clang::no_destroy]] static auto subscription = processor_.attachToCurrentRunLoop();
available_.store(true, std::memory_order_release);
});
}
#endif
}
bool available() const noexcept {
#if KONAN_HAS_FOUNDATION_FRAMEWORK
return available_.load(std::memory_order_acquire);
#else
return false;
#endif
}
void schedule(FinalizerQueue tasks, uint64_t epoch) noexcept {
if (FinalizerQueueTraits::isEmpty(tasks)) return;
#if KONAN_HAS_FOUNDATION_FRAMEWORK
RuntimeAssert(available(), "MainThreadFinalizerProcessor is unavailable");
processor_.schedule(std::move(tasks), epoch);
#else
RuntimeAssert(false, "MainThreadFinalizerProcessor is unavailable");
#endif
}
template <typename F>
std::invoke_result_t<F, RunLoopFinalizerProcessorConfig&> withConfig(F&& f) noexcept {
#if KONAN_HAS_FOUNDATION_FRAMEWORK
return processor_.withConfig(std::forward<F>(f));
#else
RuntimeAssert(false, "MainThreadFinalizerProcessor is unavailable");
#endif
}
private:
#if KONAN_HAS_FOUNDATION_FRAMEWORK
std::atomic<bool> available_ = false;
RunLoopFinalizerProcessor<FinalizerQueue, FinalizerQueueTraits> processor_;
#endif
};
} // namespace kotlin::alloc
kotlin-native/runtime/src/alloc/common/cpp/RunLoopFinalizerProcessor.hpp
+8 -6
View File
@@ -18,6 +18,8 @@
namespace kotlin::alloc {
// Configuration for `RunLoopFinalizerProcessor`
// When updating the default values, do not forget to update stdlib API docs.
struct RunLoopFinalizerProcessorConfig {
// How long can finalizers be processed in a single task. If some finalizer takes too long, the entire
// batch of `batchSize` will overshoot this target.
@@ -27,7 +29,7 @@ struct RunLoopFinalizerProcessorConfig {
// This cannot be too small to allow the attached run loop process other tasks (not from this finalizer processor).
std::chrono::nanoseconds minTimeBetweenTasks = std::chrono::milliseconds(1);
// How many finalizers are processed in a single batch in a single autoreleasepool.
size_t batchSize = 100;
uint64_t batchSize = 100;
};
#if KONAN_HAS_FOUNDATION_FRAMEWORK
@@ -109,7 +111,7 @@ private:
}
}
steady_clock::time_point deadline;
size_t batchCount;
uint64_t batchCount;
{
std::unique_lock guard(configMutex_);
RuntimeLogDebug(
@@ -118,14 +120,14 @@ private:
deadline = startTime + config_.maxTimeInTask;
batchCount = config_.batchSize;
}
size_t processedCount = 0;
uint64_t processedCount = 0;
while (true) {
auto now = steady_clock::now();
if (now > deadline) {
// Finalization is being run too long. Stop processing and reschedule until the next allowed time.
std::unique_lock guard(configMutex_);
RuntimeLogDebug(
{kTagGC}, "Processing %zu finalizers on a run loop has taken %" PRId64 " ms. Stopping for %" PRId64 "ms.",
{kTagGC}, "Processing %" PRIu64 " finalizers on a run loop has taken %" PRId64 " ms. Stopping for %" PRId64 "ms.",
processedCount, std::chrono::duration_cast<milliseconds>(now - startTime).count().value,
std::chrono::duration_cast<std::chrono::milliseconds>(config_.minTimeBetweenTasks).count());
timer_.setNextFiring(config_.minTimeBetweenTasks);
@@ -134,7 +136,7 @@ private:
}
{
objc_support::AutoreleasePool autoreleasePool;
for (size_t i = 0; i < batchCount; ++i) {
for (uint64_t i = 0; i < batchCount; ++i) {
// There's no point checking `deadline` here since the majority of the time will probably
// be spent in `AutoreleasePool` destructor.
if (!FinalizerQueueTraits::processSingle(currentQueue_.queue)) {
@@ -154,7 +156,7 @@ private:
// would definitely have to wait long enough to see the updated lastProcessTimestamp_.
lastProcessTimestamp_ = steady_clock::now();
RuntimeLogDebug(
{kTagGC}, "Processing %zu finalizers on a run loop has finished in %" PRId64 "ms.", processedCount,
{kTagGC}, "Processing %" PRIu64 " finalizers on a run loop has finished in %" PRId64 "ms.", processedCount,
std::chrono::duration_cast<milliseconds>(lastProcessTimestamp_ - startTime).count().value);
return;
}
kotlin-native/runtime/src/alloc/common/cpp/SegregatedFinalizerQueue.hpp
+32
View File
@@ -0,0 +1,32 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <cstddef>
#include <utility>
namespace kotlin::alloc {
// Finalizer queue split between threads.
template <typename FinalizerQueue>
struct SegregatedFinalizerQueue {
FinalizerQueue regular;
FinalizerQueue mainThread;
size_t size() noexcept { return regular.size() + mainThread.size(); }
void mergeIntoRegular() noexcept {
regular.TransferAllFrom(std::move(mainThread));
mainThread = FinalizerQueue{};
}
void mergeFrom(SegregatedFinalizerQueue rhs) noexcept {
regular.TransferAllFrom(std::move(rhs.regular));
mainThread.TransferAllFrom(std::move(rhs.mainThread));
}
};
} // namespace kotlin::alloc
kotlin-native/runtime/src/alloc/custom/cpp/CustomFinalizerProcessor.hpp
+16 -7
View File
@@ -6,22 +6,29 @@
#ifndef CUSTOM_ALLOC_CPP_CUSTOMFINALIZERPROCESSOR_HPP_
#define CUSTOM_ALLOC_CPP_CUSTOMFINALIZERPROCESSOR_HPP_
#include "Allocator.hpp"
#include "AtomicStack.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "ExtraObjectCell.hpp"
#include "FinalizerHooks.hpp"
#include "SegregatedFinalizerQueue.hpp"
namespace kotlin::alloc {
using FinalizerQueue = kotlin::alloc::AtomicStack<kotlin::alloc::ExtraObjectCell>;
using FinalizerQueueSingle = AtomicStack<ExtraObjectCell>;
using FinalizerQueue = SegregatedFinalizerQueue<FinalizerQueueSingle>;
struct FinalizerQueueTraits {
static bool isEmpty(const FinalizerQueue& queue) noexcept { return queue.isEmpty(); }
static bool isEmpty(const FinalizerQueueSingle& queue) noexcept { return queue.isEmpty(); }
static void add(FinalizerQueue& into, FinalizerQueue from) noexcept { into.TransferAllFrom(std::move(from)); }
static void add(FinalizerQueueSingle& into, FinalizerQueueSingle from) noexcept { into.TransferAllFrom(std::move(from)); }
static void process(FinalizerQueue queue) noexcept {
while (auto* cell = queue.Pop()) {
static void process(FinalizerQueueSingle queue) noexcept {
while (processSingle(queue)) {
}
}
static bool processSingle(FinalizerQueueSingle& queue) noexcept {
if (auto* cell = queue.Pop()) {
auto* extraObject = cell->Data();
if (auto* baseObject = extraObject->GetBaseObject()) {
RunFinalizers(baseObject);
@@ -30,7 +37,9 @@ struct FinalizerQueueTraits {
// that the only finalization step is destroying it.
destroyExtraObjectData(*extraObject);
}
return true;
}
return false;
}
};
kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectCell.hpp
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include "ExtraObjectData.hpp"
namespace kotlin::alloc {
struct ExtraObjectCell {
mm::ExtraObjectData* Data() { return reinterpret_cast<mm::ExtraObjectData*>(data_); }
// This is used to simultaneously build two lists: a free list and a finalizers queue.
// A cell cannot exist in both of them, but can be in neither when it's alive.
std::atomic<ExtraObjectCell*> next_;
struct alignas(mm::ExtraObjectData) {
uint8_t data_[sizeof(mm::ExtraObjectData)];
};
static ExtraObjectCell* fromExtraObject(mm::ExtraObjectData* extraObjectData) {
return reinterpret_cast<ExtraObjectCell*>(reinterpret_cast<uint8_t*>(extraObjectData) - offsetof(ExtraObjectCell, data_));
}
};
} // namespace kotlin::alloc
kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp
+2 -17
View File
@@ -11,28 +11,13 @@
#include <cstdint>
#include "AtomicStack.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "ExtraObjectCell.hpp"
#include "ExtraObjectData.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
struct ExtraObjectCell {
mm::ExtraObjectData* Data() { return reinterpret_cast<mm::ExtraObjectData*>(data_); }
// This is used to simultaneously build two lists: a free list and a finalizers queue.
// A cell cannot exist in both of them, but can be in neither when it's alive.
std::atomic<ExtraObjectCell*> next_;
struct alignas(mm::ExtraObjectData) {
uint8_t data_[sizeof(mm::ExtraObjectData)];
};
static ExtraObjectCell* fromExtraObject(mm::ExtraObjectData* extraObjectData) {
return reinterpret_cast<ExtraObjectCell*>(reinterpret_cast<uint8_t*>(extraObjectData) - offsetof(ExtraObjectCell, data_));
}
};
using FinalizerQueue = AtomicStack<ExtraObjectCell>;
class alignas(8) ExtraObjectPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepExtraObjectsScope;
kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPageTest.cpp
+2 -3
View File
@@ -17,7 +17,6 @@ namespace {
using Data = typename kotlin::mm::ExtraObjectData;
using Cell = typename kotlin::alloc::ExtraObjectCell;
using Page = typename kotlin::alloc::ExtraObjectPage;
using Queue = typename kotlin::alloc::AtomicStack<Cell>;
Data* alloc(Page* page) {
Data* ptr = page->TryAllocate();
@@ -40,7 +39,7 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
Page* page = Page::Create(0);
Queue finalizerQueue;
kotlin::alloc::FinalizerQueue finalizerQueue;
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweepExtraObjects();
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
@@ -57,7 +56,7 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
++count;
}
EXPECT_EQ(count, EXTRA_OBJECT_COUNT);
Queue finalizerQueue;
kotlin::alloc::FinalizerQueue finalizerQueue;
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweepExtraObjects();
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp
+6 -1
View File
@@ -49,7 +49,12 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
extraObject->ClearRegularWeakReferenceImpl();
CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject));
finalizerQueue.Push(ExtraObjectCell::fromExtraObject(extraObject));
auto* cell = ExtraObjectCell::fromExtraObject(extraObject);
if (compiler::objcDisposeOnMain() && extraObject->getFlag(mm::ExtraObjectData::FLAGS_RELEASE_ON_MAIN_QUEUE)) {
finalizerQueue.mainThread.Push(cell);
} else {
finalizerQueue.regular.Push(cell);
}
if (HasFinalizersDataInObject(heapObjHeader->object())) {
// The object must survive until the finalizers for it are finished.
gcHandle.addMarkedObject();
kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp
+1 -1
View File
@@ -80,7 +80,7 @@ ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexce
void Heap::AddToFinalizerQueue(FinalizerQueue queue) noexcept {
std::unique_lock guard(pendingFinalizerQueueMutex_);
pendingFinalizerQueue_.TransferAllFrom(std::move(queue));
pendingFinalizerQueue_.mergeFrom(std::move(queue));
}
FinalizerQueue Heap::ExtractFinalizerQueue() noexcept {
kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.hpp
+6 -4
View File
@@ -9,11 +9,12 @@
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "ObjectFactory.hpp"
#include "ObjectFactoryAllocator.hpp"
#include "ObjectFactorySweep.hpp"
#include "Logging.hpp"
namespace kotlin::alloc {
@@ -32,6 +33,10 @@ struct ObjectFactoryTraits {
using ObjectFactoryImpl = ObjectFactory<ObjectFactoryTraits>;
using FinalizerQueueSingle = ObjectFactoryImpl::FinalizerQueue;
using FinalizerQueue = SegregatedFinalizerQueue<FinalizerQueueSingle>;
using FinalizerQueueTraits = ObjectFactoryImpl::FinalizerQueueTraits;
class Allocator::Impl : private Pinned {
public:
Impl() noexcept = default;
@@ -59,7 +64,4 @@ private:
ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
};
using FinalizerQueue = ObjectFactoryImpl::FinalizerQueue;
using FinalizerQueueTraits = ObjectFactoryImpl::FinalizerQueueTraits;
} // namespace kotlin::alloc
kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactory.hpp
+24 -3
View File
@@ -325,6 +325,20 @@ public:
AssertCorrect();
}
unique_ptr<Node> Pop() noexcept {
AssertCorrect();
auto result = std::move(root_);
if (result) {
--size_;
root_ = std::move(result->next_);
if (last_ == result.get()) {
last_ = nullptr;
}
}
AssertCorrect();
return result;
}
private:
friend class ObjectFactoryStorage;
@@ -679,10 +693,15 @@ public:
}
}
void MergeWith(FinalizerQueue &&other) {
consumer_.MergeWith(std::move(other.consumer_));
bool FinalizeSingle() noexcept {
if (auto node = consumer_.Pop()) {
RunFinalizers(NodeRef(*node).GetObjHeader());
}
return false;
}
void TransferAllFrom(FinalizerQueue&& other) { consumer_.MergeWith(std::move(other.consumer_)); }
Iterable IterForTests() noexcept { return Iterable(*this); }
private:
@@ -694,9 +713,11 @@ public:
struct FinalizerQueueTraits {
static bool isEmpty(const FinalizerQueue& queue) noexcept { return queue.size() == 0; }
static void add(FinalizerQueue& into, FinalizerQueue from) noexcept { into.MergeWith(std::move(from)); }
static void add(FinalizerQueue& into, FinalizerQueue from) noexcept { into.TransferAllFrom(std::move(from)); }
static void process(FinalizerQueue queue) noexcept { queue.Finalize(); }
static bool processSingle(FinalizerQueue& queue) noexcept { return queue.FinalizeSingle(); }
};
class Iterable {
kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp
+13 -5
View File
@@ -5,10 +5,11 @@
#pragma once
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerHooks.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "SegregatedFinalizerQueue.hpp"
namespace kotlin::alloc {
@@ -43,8 +44,9 @@ void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory
}
template <typename Traits>
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
SegregatedFinalizerQueue<typename Traits::ObjectFactory::FinalizerQueue> Sweep(
gc::GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
SegregatedFinalizerQueue<typename Traits::ObjectFactory::FinalizerQueue> finalizerQueue;
auto sweepHandle = handle.sweep();
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
@@ -56,7 +58,12 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typena
}
sweepHandle.addSweptObject();
if (HasFinalizers(objHeader)) {
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
auto* extraObject = mm::ExtraObjectData::Get(objHeader);
if (compiler::objcDisposeOnMain() && extraObject->getFlag(mm::ExtraObjectData::FLAGS_RELEASE_ON_MAIN_QUEUE)) {
objectFactoryIter.MoveAndAdvance(finalizerQueue.mainThread, it);
} else {
objectFactoryIter.MoveAndAdvance(finalizerQueue.regular, it);
}
} else {
objectFactoryIter.EraseAndAdvance(it);
}
@@ -66,7 +73,8 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typena
}
template <typename Traits>
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
SegregatedFinalizerQueue<typename Traits::ObjectFactory::FinalizerQueue> Sweep(
gc::GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
auto iter = objectFactory.LockForIter();
return Sweep<Traits>(handle, iter);
}
kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp
+3 -2
View File
@@ -198,11 +198,12 @@ public:
gc::processWeaks<ProcessWeakTraits>(gc::GCHandle::getByEpoch(0), specialRefRegistry_);
alloc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
auto finalizers = alloc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
finalizers.mergeIntoRegular();
std::vector<ObjHeader*> objects;
for (auto node : finalizers.IterForTests()) {
for (auto node : finalizers.regular.IterForTests()) {
objects.push_back(node.GetObjHeader());
}
finalizers_.push_back(std::move(finalizers));
finalizers_.push_back(std::move(finalizers.regular));
return objects;
}
kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp
+34
View File
@@ -489,6 +489,40 @@ TEST(ObjectFactoryStorageTest, MoveAll) {
EXPECT_THAT(consumer.size(), 3);
}
TEST(ObjectFactoryStorageTest, Pop) {
ObjectFactoryStorageRegular storage;
Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
Consumer<ObjectFactoryStorageRegular> consumer;
producer.Insert<int>(1);
producer.Insert<int>(2);
producer.Insert<int>(3);
producer.Publish();
{
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
}
}
std::vector<int> popped;
while (auto element = consumer.Pop()) {
popped.push_back(element->Data<int>());
}
auto actual = Collect<int>(storage);
auto actualConsumer = Collect<int, alignof(void*)>(consumer);
EXPECT_THAT(actual, testing::IsEmpty());
EXPECT_THAT(actualConsumer, testing::IsEmpty());
EXPECT_THAT(popped, testing::ElementsAre(1, 2, 3));
EXPECT_THAT(storage.GetSizeUnsafe(), 0);
EXPECT_THAT(producer.size(), 0);
EXPECT_THAT(consumer.size(), 0);
}
TEST(ObjectFactoryStorageTest, MergeWith) {
ObjectFactoryStorageRegular storage;
Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+7 -3
View File
@@ -181,17 +181,21 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
// also sweeps extraObjects
auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle);
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
}
finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(allocator_.impl().heap().ExtractFinalizerQueue());
#endif
scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread());
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
if (!mainThreadFinalizerProcessor_.available()) {
finalizerQueue.mergeIntoRegular();
}
// This may start a new thread. On some pthreads implementations, this may block waiting for concurrent thread
// destructors running. So, it must ensured that no locks are held by this point.
// TODO: Consider having an always on sleeping finalizer thread.
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch);
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue.regular), epoch);
mainThreadFinalizerProcessor_.schedule(std::move(finalizerQueue.mainThread), epoch);
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+6 -1
View File
@@ -17,6 +17,7 @@
#include "GCState.hpp"
#include "GCStatistics.hpp"
#include "IntrusiveList.hpp"
#include "MainThreadFinalizerProcessor.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ObjectData.hpp"
#include "ConcurrentMark.hpp"
@@ -72,6 +73,9 @@ public:
bool FinalizersThreadIsRunning() noexcept;
GCStateHolder& state() noexcept { return state_; }
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits>& mainThreadFinalizerProcessor() noexcept {
return mainThreadFinalizerProcessor_;
}
private:
void mainGCThreadBody();
@@ -81,7 +85,8 @@ private:
gcScheduler::GCScheduler& gcScheduler_;
GCStateHolder state_;
FinalizerProcessor<alloc::FinalizerQueue, alloc::FinalizerQueueTraits> finalizerProcessor_;
FinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> finalizerProcessor_;
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> mainThreadFinalizerProcessor_;
mark::ConcurrentMark markDispatcher_;
ScopedThread mainThread_;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+8
View File
@@ -79,6 +79,14 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinalized(epoch);
}
void gc::GC::configureMainThreadFinalizerProcessor(std::function<void(alloc::RunLoopFinalizerProcessorConfig&)> f) noexcept {
impl_->gc().mainThreadFinalizerProcessor().withConfig(std::move(f));
}
bool gc::GC::mainThreadFinalizerProcessorAvailable() noexcept {
return impl_->gc().mainThreadFinalizerProcessor().available();
}
ALWAYS_INLINE void gc::beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* value) noexcept {
barriers::beforeHeapRefUpdate(ref, value);
}
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+4
View File
@@ -12,6 +12,7 @@
#include "ExtraObjectData.hpp"
#include "GCScheduler.hpp"
#include "ReferenceOps.hpp"
#include "RunLoopFinalizerProcessor.hpp"
#include "Utils.hpp"
namespace kotlin {
@@ -78,6 +79,9 @@ public:
void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept;
void configureMainThreadFinalizerProcessor(std::function<void(alloc::RunLoopFinalizerProcessorConfig&)> f) noexcept;
bool mainThreadFinalizerProcessorAvailable() noexcept;
private:
std::unique_ptr<Impl> impl_;
};
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+6
View File
@@ -56,6 +56,12 @@ void gc::GC::WaitFinished(int64_t epoch) noexcept {}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {}
void gc::GC::configureMainThreadFinalizerProcessor(std::function<void(alloc::RunLoopFinalizerProcessorConfig&)> f) noexcept {}
bool gc::GC::mainThreadFinalizerProcessorAvailable() noexcept {
return false;
}
ALWAYS_INLINE void gc::beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* value) noexcept {}
ALWAYS_INLINE OBJ_GETTER(gc::weakRefReadBarrier, std::atomic<ObjHeader*>& weakReferee) noexcept {
kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp
+8
View File
@@ -79,6 +79,14 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinalized(epoch);
}
void gc::GC::configureMainThreadFinalizerProcessor(std::function<void(alloc::RunLoopFinalizerProcessorConfig&)> f) noexcept {
impl_->gc().mainThreadFinalizerProcessor().withConfig(std::move(f));
}
bool gc::GC::mainThreadFinalizerProcessorAvailable() noexcept {
return impl_->gc().mainThreadFinalizerProcessor().available();
}
ALWAYS_INLINE void gc::beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* value) noexcept {}
ALWAYS_INLINE OBJ_GETTER(gc::weakRefReadBarrier, std::atomic<ObjHeader*>& weakReferee) noexcept {
kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp
+7 -3
View File
@@ -204,19 +204,23 @@ void gc::ParallelMarkConcurrentSweep::PerformFullGC(int64_t epoch) noexcept {
// also sweeps extraObjects
auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle);
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
}
finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(allocator_.impl().heap().ExtractFinalizerQueue());
#endif
scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread());
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
if (!mainThreadFinalizerProcessor_.available()) {
finalizerQueue.mergeIntoRegular();
}
// This may start a new thread. On some pthreads implementations, this may block waiting for concurrent thread
// destructors running. So, it must ensured that no locks are held by this point.
// TODO: Consider having an always on sleeping finalizer thread.
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch);
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue.regular), epoch);
mainThreadFinalizerProcessor_.schedule(std::move(finalizerQueue.mainThread), epoch);
}
void gc::ParallelMarkConcurrentSweep::reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept {
kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.hpp
+6 -1
View File
@@ -17,6 +17,7 @@
#include "GCState.hpp"
#include "GCStatistics.hpp"
#include "IntrusiveList.hpp"
#include "MainThreadFinalizerProcessor.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ObjectData.hpp"
#include "ParallelMark.hpp"
@@ -72,6 +73,9 @@ public:
void reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, size_t auxGCThreads) noexcept;
GCStateHolder& state() noexcept { return state_; }
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits>& mainThreadFinalizerProcessor() noexcept {
return mainThreadFinalizerProcessor_;
}
private:
void mainGCThreadBody();
@@ -82,7 +86,8 @@ private:
gcScheduler::GCScheduler& gcScheduler_;
GCStateHolder state_;
FinalizerProcessor<alloc::FinalizerQueue, alloc::FinalizerQueueTraits> finalizerProcessor_;
FinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> finalizerProcessor_;
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> mainThreadFinalizerProcessor_;
mark::ParallelMark markDispatcher_;
ScopedThread mainThread_;
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+8
View File
@@ -72,6 +72,14 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinalized(epoch);
}
void gc::GC::configureMainThreadFinalizerProcessor(std::function<void(alloc::RunLoopFinalizerProcessorConfig&)> f) noexcept {
impl_->gc().mainThreadFinalizerProcessor().withConfig(std::move(f));
}
bool gc::GC::mainThreadFinalizerProcessorAvailable() noexcept {
return impl_->gc().mainThreadFinalizerProcessor().available();
}
ALWAYS_INLINE void gc::beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* value) noexcept {}
ALWAYS_INLINE OBJ_GETTER(gc::weakRefReadBarrier, std::atomic<ObjHeader*>& weakReferee) noexcept {
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+7 -3
View File
@@ -93,9 +93,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
// also sweeps extraObjects
auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle);
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
}
finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
finalizerQueue.mergeFrom(allocator_.impl().heap().ExtractFinalizerQueue());
#endif
scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread());
@@ -105,5 +105,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch);
if (!mainThreadFinalizerProcessor_.available()) {
finalizerQueue.mergeIntoRegular();
}
finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue.regular), epoch);
mainThreadFinalizerProcessor_.schedule(std::move(finalizerQueue.mainThread), epoch);
}
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+6 -1
View File
@@ -15,6 +15,7 @@
#include "GCState.hpp"
#include "GlobalData.hpp"
#include "IntrusiveList.hpp"
#include "MainThreadFinalizerProcessor.hpp"
#include "ObjectData.hpp"
#include "Types.h"
#include "Utils.hpp"
@@ -49,6 +50,9 @@ public:
bool FinalizersThreadIsRunning() noexcept;
GCStateHolder& state() noexcept { return state_; }
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits>& mainThreadFinalizerProcessor() noexcept {
return mainThreadFinalizerProcessor_;
}
private:
void PerformFullGC(int64_t epoch) noexcept;
@@ -58,7 +62,8 @@ private:
GCStateHolder state_;
ScopedThread gcThread_;
FinalizerProcessor<alloc::FinalizerQueue, alloc::FinalizerQueueTraits> finalizerProcessor_;
FinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> finalizerProcessor_;
alloc::MainThreadFinalizerProcessor<alloc::FinalizerQueueSingle, alloc::FinalizerQueueTraits> mainThreadFinalizerProcessor_;
MarkQueue markQueue_;
};
kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt
+105
View File
@@ -321,6 +321,111 @@ public object GC {
@Deprecated("No-op in modern GC implementation")
public external fun findCycle(root: Any): Array<Any>?
/**
* This objects allows to customize the behavior of the finalizer processor that runs on the main thread.
*
* On Apple platforms Kotlin/Native releases ObjC/Swift objects that were passed to Kotlin when it processes the finalizers after GC.
* Kotlin/Native can also utilize main run loop to release objects that were passed to Kotlin on the main thread. This can be turned
* off by setting `objcDisposeOnMain` binary option to `false`.
* For more information, see [iOS integration](https://kotlinlang.org/docs/native-ios-integration.html#deinitializers).
*
* This finalizer processor works as follows:
* - Finalizers that must be run on the main thread get scheduled by the GC after the main GC phase is finished
* - A task will be posted on the main run loop in which the scheduled finalizers will start processing
* - Finalizers will be processed inside an `autoreleasepool` in batches of size [batchSize]
* - If at some point during task the processor detected that more than [maxTimeInTask] has passed, it will
* stop and post another task to the main thread to continue processing finalizers later. Note that if some
* finalizer takes a very long time, the task will still process the entire [batchSize] and may significantly overflow [maxTimeInTask]
* - It's guaranteed that the time interval between tasks is at least [minTimeBetweenTasks].
*
* [maxTimeInTask] and [minTimeBetweenTasks] allow other tasks posted on the main thread (e.g. UI events) be processed without
* significant delays.
*/
public object MainThreadFinalizerProcessor {
/**
* `true` if Kotlin/Native will use [MainThreadFinalizerProcessor] to process finalizers.
*
* __Note__: at the very start of the program this will be `false`, but may turn `true` later, if Kotlin/Native determines that the
* main run loop is being processed.
*/
public val available: Boolean
get() = isAvailable()
/**
* How much time can each task take.
*
* There is no guarantee that the task will be completed under this time, this is only a hint.
*
* Setting this value too high makes some other main thread tasks (e.g. UI events) be processed with high delays.
* Setting this value too low makes ObjC/Swift objects be released with high delays which contributes to memory usage.
*
* Default: 5ms
*
* @throws [IllegalArgumentException] when value is negative.
*/
public var maxTimeInTask: Duration
get() = getMaxTimeInTask().microseconds
set(value) {
require(!value.isNegative()) { "mainThreadMaxTimeInTask must not be negative: $value" }
setMaxTimeInTask(value.inWholeMicroseconds)
}
/**
* The minimum interval between two tasks.
*
* Setting this value too high makes ObjC/Swift objects be released with high delays which contributes to memory usage.
* Setting this value too low makes some other main thread tasks (e.g. UI events) be processed with high delays.
*
* Default: 10ms
*
* @throws [IllegalArgumentException] when value is negative.
*/
public var minTimeBetweenTasks: Duration
get() = getMinTimeBetweenTasks().microseconds
set(value) {
require(!value.isNegative()) { "mainThreadMinTimeBetweenTasks must not be negative: $value" }
setMinTimeBetweenTasks(value.inWholeMicroseconds)
}
/**
* How many finalizers will be processed inside a single `autoreleasepool`.
*
* Setting this value too high makes it more likely that [maxTimeInTask] will not be respected.
* Setting this value too low makes each single finalizer take more time and so fewer finalizers will be processed in one task.
*
* Default: 100
*
* @throws [IllegalArgumentException] when value is 0.
*/
public var batchSize: ULong
get() = getBatchSize()
set(value) {
require(value > 0U) { "mainThreadBatchSize must not be 0" }
setBatchSize(value)
}
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_isAvailable")
private external fun isAvailable(): Boolean
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getMaxTimeInTask")
private external fun getMaxTimeInTask(): Long
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setMaxTimeInTask")
private external fun setMaxTimeInTask(value: Long)
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getMinTimeBetweenTasks")
private external fun getMinTimeBetweenTasks(): Long
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setMinTimeBetweenTasks")
private external fun setMinTimeBetweenTasks(value: Long)
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getBatchSize")
private external fun getBatchSize(): ULong
@GCUnsafeCall("Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setBatchSize")
private external fun setBatchSize(value: ULong)
}
@GCUnsafeCall("Kotlin_native_internal_GC_getThreshold")
private external fun getThreshold(): Int
kotlin-native/runtime/src/mm/cpp/Memory.cpp
+41
View File
@@ -438,6 +438,47 @@ extern "C" void Kotlin_native_internal_GC_setCyclicCollector(ObjHeader* gc, bool
// Nothing to do.
}
extern "C" KBoolean Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_isAvailable(ObjHeader* gc) {
return mm::GlobalData::Instance().gc().mainThreadFinalizerProcessorAvailable();
}
extern "C" KLong Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getMaxTimeInTask(ObjHeader* gc) {
KLong result;
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor([&](auto& config) noexcept -> void {
result = std::chrono::duration_cast<std::chrono::microseconds>(config.maxTimeInTask).count();
});
return result;
}
extern "C" void Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setMaxTimeInTask(ObjHeader* gc, KLong value) {
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor(
[=](auto& config) noexcept -> void { config.maxTimeInTask = std::chrono::microseconds(value); });
}
extern "C" KLong Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getMinTimeBetweenTasks(ObjHeader* gc) {
KLong result;
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor([&](auto& config) noexcept -> void {
result = std::chrono::duration_cast<std::chrono::microseconds>(config.minTimeBetweenTasks).count();
});
return result;
}
extern "C" void Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setMinTimeBetweenTasks(ObjHeader* gc, KLong value) {
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor(
[=](auto& config) noexcept -> void { config.minTimeBetweenTasks = std::chrono::microseconds(value); });
}
extern "C" KULong Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_getBatchSize(ObjHeader* gc) {
KULong result;
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor(
[&](auto& config) noexcept -> void { result = config.batchSize; });
return result;
}
extern "C" void Kotlin_native_runtime_GC_MainThreadFinalizerProcessor_setBatchSize(ObjHeader* gc, KULong value) {
mm::GlobalData::Instance().gc().configureMainThreadFinalizerProcessor([=](auto& config) noexcept -> void { config.batchSize = value; });
}
extern "C" bool Kotlin_Any_isShareable(ObjHeader* thiz) {
// TODO: Remove when legacy MM is gone.
return true;