azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Refactor GC logging and statistics

Browse Source 
Same code is now used for GC logging in all gc versions.

^KT-53064
This commit is contained in:
Pavel Kunyavskiy
2022-10-04 14:57:21 +02:00
committed by Space Team
parent c4e2901a1d
commit 0959255379
22 changed files with 646 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+35 -59
View File
@@ -20,6 +20,7 @@
#include "ThreadSuspension.hpp"
#include "GCState.hpp"
#include "FinalizerProcessor.hpp"
#include "GCStatistics.hpp"
using namespace kotlin;
@@ -27,6 +28,7 @@ namespace {
[[clang::no_destroy]] std::mutex markingMutex;
[[clang::no_destroy]] std::condition_variable markingCondVar;
[[clang::no_destroy]] std::atomic<bool> markingRequested = false;
[[clang::no_destroy]] std::atomic<uint64_t> markingEpoch = 0;
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
@@ -72,25 +74,28 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
std::unique_lock lock(markingMutex);
if (!markingRequested.load())
return;
if (!markingRequested.load()) return;
AutoReset scopedAssignMarking(&marking_, true);
threadData_.Publish();
markingCondVar.wait(lock, []() { return !markingRequested.load(); });
// // Unlock while marking to allow mutliple threads to mark in parallel.
lock.unlock();
RuntimeLogDebug({kTagGC}, "Parallel marking in thread %d", konan::currentThreadId());
uint64_t epoch = markingEpoch.load();
GCLogDebug(epoch, "Parallel marking in thread %d", konan::currentThreadId());
MarkQueue markQueue;
gc::collectRootSetForThread<internal::MarkTraits>(markQueue, threadData_);
MarkStats stats = gc::Mark<internal::MarkTraits>(markQueue);
gc_.MergeMarkStats(stats);
auto handle = GCHandle::getByEpoch(epoch);
gc::collectRootSetForThread<internal::MarkTraits>(handle, markQueue, threadData_);
gc::Mark<internal::MarkTraits>(handle, markQueue);
}
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory),
gcScheduler_(gcScheduler),
finalizerProcessor_(std_support::make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch); })) {
finalizerProcessor_(std_support::make_unique<FinalizerProcessor>([this](int64_t epoch) {
state_.finalized(epoch);
GCHandle::getByEpoch(epoch).finalizersDone();
})) {
gcScheduler_.SetScheduleGC([this]() NO_INLINE {
RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
// This call acquires a lock, so we need to ensure that we're in the safe state.
@@ -135,76 +140,46 @@ void gc::ConcurrentMarkAndSweep::SetMarkingBehaviorForTests(MarkingBehavior mark
}
bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
SetMarkingRequested();
auto timeStartUs = konan::getTimeMicros();
auto gcHandle = GCHandle::create(epoch);
SetMarkingRequested(epoch);
bool didSuspend = mm::RequestThreadsSuspension();
RuntimeAssert(didSuspend, "Only GC thread can request suspension");
RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
gcHandle.suspensionRequested();
RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "Concurrent GC must run on unregistered thread");
WaitForThreadsReadyToMark();
auto timeSuspendUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
lastGCMarkStats_ = MarkStats();
gcHandle.threadsAreSuspended();
auto& scheduler = gcScheduler_;
scheduler.gcData().OnPerformFullGC();
state_.start(epoch);
RuntimeLogInfo(
{kTagGC}, "Started GC epoch %" PRId64 ". Time since last GC %" PRIu64 " microseconds", epoch, timeStartUs - lastGCTimestampUs_);
CollectRootSetAndStartMarking();
CollectRootSetAndStartMarking(gcHandle);
// Can be unsafe, because we've stopped the world.
auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
MergeMarkStats(markStats);
RuntimeLogDebug({kTagGC}, "Waiting for marking in threads");
mm::WaitForThreadsSuspension();
auto timeMarkingUs = konan::getTimeMicros();
RuntimeLogInfo({kTagGC}, "Collected root set of size %zu and marked %zu objects in all threads in %" PRIu64 " microseconds", lastGCMarkStats_.rootSetSize, lastGCMarkStats_.aliveHeapSet, timeMarkingUs - timeSuspendUs);
mm::ExtraObjectDataFactory& extraObjectDataFactory = mm::GlobalData::Instance().extraObjectDataFactory();
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
scheduler.gcData().UpdateAliveSetBytes(lastGCMarkStats_.aliveHeapSetBytes);
gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
auto timeSweepExtraObjectsUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkingUs);
gc::SweepExtraObjects<SweepTraits>(gcHandle, extraObjectDataFactory);
auto objectFactoryIterable = objectFactory_.LockForIter();
mm::ResumeThreads();
auto timeResumeUs = konan::getTimeMicros();
gcHandle.threadsAreResumed();
RuntimeLogInfo(
{kTagGC}, "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %" PRIu64 " microseconds",
timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
auto finalizerQueue = gc::Sweep<SweepTraits>(objectFactoryIterable);
auto timeSweepUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs);
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, objectFactoryIterable);
kotlin::compactObjectPoolInMainThread();
// Can be unsafe, because we have a lock in objectFactoryIterable
auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
auto finalizersCount = finalizerQueue.size();
auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
finalizerProcessor_->ScheduleTasks(std::move(finalizerQueue), epoch);
RuntimeLogInfo(
{kTagGC},
"Finished GC epoch %" PRId64 ". Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
" microseconds",
epoch, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeSweepUs - timeStartUs);
lastGCTimestampUs_ = timeResumeUs;
return true;
}
@@ -234,8 +209,9 @@ namespace {
}
} // namespace
void gc::ConcurrentMarkAndSweep::SetMarkingRequested() noexcept {
void gc::ConcurrentMarkAndSweep::SetMarkingRequested(uint64_t epoch) noexcept {
markingRequested = markingBehavior_ == MarkingBehavior::kMarkOwnStack;
markingEpoch = epoch;
}
void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept {
@@ -244,16 +220,16 @@ void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept {
}
}
NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking() noexcept {
NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept {
std::unique_lock lock(markingMutex);
markingRequested = false;
gc::collectRootSet<internal::MarkTraits>(
markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); });
gcHandle,
markQueue_,
[](mm::ThreadData& thread) {
return !thread.gc().impl().gc().marking_.load();
}
);
RuntimeLogDebug({kTagGC}, "Requesting marking in threads");
markingCondVar.notify_all();
}
void gc::ConcurrentMarkAndSweep::MergeMarkStats(gc::MarkStats stats) noexcept {
std::unique_lock lock(markingMutex);
lastGCMarkStats_.merge(stats);
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+3 -5
View File
@@ -18,6 +18,7 @@
#include "Utils.hpp"
#include "GCState.hpp"
#include "std_support/Memory.hpp"
#include "GCStatistics.hpp"
namespace kotlin {
@@ -114,25 +115,22 @@ public:
void StopFinalizerThreadIfRunning() noexcept;
bool FinalizersThreadIsRunning() noexcept;
void SetMarkingBehaviorForTests(MarkingBehavior markingBehavior) noexcept;
void SetMarkingRequested() noexcept;
void SetMarkingRequested(uint64_t epoch) noexcept;
void WaitForThreadsReadyToMark() noexcept;
void CollectRootSetAndStartMarking() noexcept;
void CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept;
private:
// Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
bool PerformFullGC(int64_t epoch) noexcept;
void MergeMarkStats(MarkStats stats) noexcept;
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
GCScheduler& gcScheduler_;
uint64_t lastGCTimestampUs_ = 0;
GCStateHolder state_;
ScopedThread gcThread_;
std_support::unique_ptr<FinalizerProcessor> finalizerProcessor_;
MarkQueue markQueue_;
MarkStats lastGCMarkStats_;
MarkingBehavior markingBehavior_;
};
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+2 -2
View File
@@ -1172,7 +1172,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
std_support::vector<std::future<void>> gcFutures(kDefaultThreadCount);
mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested();
mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested(0);
for (int i = 0; i < kDefaultThreadCount; ++i) {
gcFutures[i] = mutators[i]
@@ -1181,7 +1181,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
if (GetParam() == gc::ConcurrentMarkAndSweep::kMarkOwnStack) {
mm::GlobalData::Instance().gc().impl().gc().WaitForThreadsReadyToMark();
mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking();
mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking(gc::GCHandle::createFakeForTests());
}
for (int i = 0; i < kDefaultThreadCount; ++i) {
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+16
View File
@@ -6,6 +6,7 @@
#include "GCImpl.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ThreadSuspension.hpp"
#include "std_support/Memory.hpp"
@@ -74,6 +75,20 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
return impl_->objectFactory().GetObjectsCountUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
}
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
return impl_->gcScheduler().config();
}
@@ -81,6 +96,7 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
void gc::GC::ClearForTests() noexcept {
impl_->gc().StopFinalizerThreadIfRunning();
impl_->objectFactory().ClearForTests();
GCHandle::ClearForTests();
}
void gc::GC::StartFinalizerThreadIfNeeded() noexcept {
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+5
View File
@@ -58,6 +58,11 @@ public:
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
size_t GetHeapObjectsCountUnsafe() const noexcept;
size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
size_t GetExtraObjectsCountUnsafe() const noexcept;
size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
void ClearForTests() noexcept;
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp
+314
View File
@@ -0,0 +1,314 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
#include "GCStatistics.hpp"
#include "Mutex.hpp"
#include "Porting.h"
#include "Types.h"
#include "Logging.hpp"
#include "ThreadData.hpp"
#include "std_support/Optional.hpp"
#include <cinttypes>
#include <mutex>
namespace {
struct MemoryUsageMap {
std::optional<kotlin::gc::MemoryUsage> heap;
std::optional<kotlin::gc::MemoryUsage> extra;
};
struct RootSetStatistics {
KLong threadLocalReferences;
KLong stackReferences;
KLong globalReferences;
KLong stableReferences;
KLong total() const { return threadLocalReferences + stableReferences + globalReferences + stableReferences; }
};
struct GCInfo {
std::optional<uint64_t> epoch;
std::optional<KLong> startTime; // time since process start
std::optional<KLong> endTime;
std::optional<KLong> pauseStartTime;
std::optional<KLong> pauseEndTime;
std::optional<KLong> finalizersDoneTime;
std::optional<RootSetStatistics> rootSet;
std::optional<kotlin::gc::MemoryUsage> markStats;
MemoryUsageMap memoryUsageBefore;
MemoryUsageMap memoryUsageAfter;
};
GCInfo last;
GCInfo current;
// This lock can be got by thread in runnable state making parallel mark, so
kotlin::SpinLock<kotlin::MutexThreadStateHandling::kIgnore> lock;
GCInfo* statByEpoch(uint64_t epoch) {
if (current.epoch == epoch) return &current;
if (last.epoch == epoch) return &last;
return nullptr;
}
} // namespace
namespace kotlin::gc {
GCHandle GCHandle::create(uint64_t epoch) {
std::lock_guard guard(lock);
RuntimeAssert(statByEpoch(epoch) == nullptr, "Starting epoch, which already existed");
if (current.epoch) {
last = current;
current = {};
RuntimeLogWarning({kTagGC}, "Starting new GC epoch, while previous is not finished\n");
}
current.epoch = static_cast<KLong>(epoch);
current.startTime = static_cast<KLong>(konan::getTimeNanos());
if (last.endTime) {
GCLogInfo(epoch, "Started. Time since last GC %" PRIu64 " microseconds.", *current.startTime - *last.endTime);
} else {
GCLogInfo(epoch, "Started.");
}
return getByEpoch(epoch);
}
GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits<uint64_t>::max()); }
GCHandle GCHandle::getByEpoch(uint64_t epoch) {
return GCHandle{epoch};
}
void GCHandle::ClearForTests() {
std::lock_guard guard(lock);
current = {};
last = {};
}
void GCHandle::finished() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->endTime = static_cast<KLong>(konan::getTimeNanos());
if (stat->rootSet) {
GCLogInfo(
epoch_,
"Root set: "
"%" PRIu64 " thread local references, "
"%" PRIu64 " stack references, "
"%" PRIu64 " global references, "
"%" PRIu64 " stable references. "
"In total %" PRIu64 " roots.",
stat->rootSet->threadLocalReferences, stat->rootSet->stackReferences, stat->rootSet->globalReferences,
stat->rootSet->stableReferences, stat->rootSet->total());
}
if (stat->startTime) {
auto time = (*current.endTime - *current.startTime) / 1000;
GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time);
}
if (stat == &current) {
last = current;
current = {};
}
}
}
void GCHandle::suspensionRequested() {
std::lock_guard guard(lock);
GCLogDebug(epoch_, "Requested thread suspension by thread %d", konan::currentThreadId());
if (auto* stat = statByEpoch(epoch_)) {
stat->pauseStartTime = static_cast<KLong>(konan::getTimeNanos());
}
}
void GCHandle::threadsAreSuspended() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (stat->pauseStartTime) {
auto time = (konan::getTimeNanos() - *stat->pauseStartTime) / 1000;
GCLogInfo(epoch_, "Suspended all threads in %" PRIu64 " microseconds", time);
return;
}
}
}
void GCHandle::threadsAreResumed() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->pauseEndTime = static_cast<KLong>(konan::getTimeNanos());
if (stat->pauseStartTime) {
auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
GCLogInfo(epoch_, "Resume all threads. Total pause time is %" PRId64 " microseconds.", time);
return;
}
}
}
void GCHandle::finalizersDone() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->finalizersDoneTime = static_cast<KLong>(konan::getTimeNanos());
if (stat->endTime) {
auto time = (*stat->finalizersDoneTime - *stat->endTime) / 1000;
GCLogInfo(epoch_, "Finalization is done in %" PRId64 " microseconds after epoch end.", time);
return;
}
}
GCLogInfo(epoch_, "Finalization is done.");
}
void GCHandle::finalizersScheduled(uint64_t finalizersCount) {
GCLogInfo(epoch_, "Finalization is scheduled for %" PRIu64 " objects.", finalizersCount);
}
void GCHandle::threadRootSetCollected(mm::ThreadData &threadData, uint64_t threadLocalReferences, uint64_t stackReferences) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (!stat->rootSet) {
stat->rootSet = RootSetStatistics{0, 0, 0, 0};
}
stat->rootSet->stackReferences += static_cast<KLong>(stackReferences);
stat->rootSet->threadLocalReferences += static_cast<KLong>(threadLocalReferences);
}
}
void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (!stat->rootSet) {
stat->rootSet = RootSetStatistics{0, 0, 0, 0};
}
stat->rootSet->globalReferences += static_cast<KLong>(globalReferences);
stat->rootSet->stableReferences += static_cast<KLong>(stableReferences);
}
}
void GCHandle::heapUsageBefore(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageBefore.heap = usage;
}
}
void GCHandle::marked(kotlin::gc::MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (!stat->markStats) {
stat->markStats = MemoryUsage{0, 0};
}
stat->markStats->totalObjectsSize += usage.totalObjectsSize;
stat->markStats->objectsCount += usage.objectsCount;
}
}
MemoryUsage GCHandle::getMarked() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (stat->markStats) {
return *stat->markStats;
}
}
return MemoryUsage{0, 0};
}
void GCHandle::heapUsageAfter(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageAfter.heap = usage;
if (stat->memoryUsageBefore.heap) {
GCLogInfo(
epoch_,
"Collected %" PRId64 " heap objects of total size %" PRId64 " bytes. "
"%" PRId64 " heap objects of total size %" PRId64 " bytes are still alive.",
stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount,
stat->memoryUsageBefore.heap->totalObjectsSize - stat->memoryUsageAfter.heap->totalObjectsSize,
stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->totalObjectsSize);
}
if (stat->markStats) {
RuntimeAssert(
stat->markStats->objectsCount == usage.objectsCount,
"Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep",
stat->markStats->objectsCount, usage.objectsCount);
RuntimeAssert(
stat->markStats->totalObjectsSize == usage.totalObjectsSize,
"Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep",
stat->markStats->totalObjectsSize, usage.totalObjectsSize);
}
}
}
void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageBefore.extra = usage;
}
}
void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageAfter.extra = usage;
if (stat->memoryUsageBefore.extra) {
GCLogInfo(
epoch_,
"Collected %" PRId64 " extra objects of total size %" PRId64 ". "
"%" PRId64 " extra objects of total size %" PRId64 " are still alive.",
stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount,
stat->memoryUsageBefore.extra->totalObjectsSize - stat->memoryUsageAfter.extra->totalObjectsSize,
stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->totalObjectsSize);
}
}
}
MemoryUsage GCHandle::GCSweepScope::getUsage() {
return MemoryUsage{
mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(),
mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(),
};
}
GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) :
handle_(handle) {
handle_.heapUsageBefore(getUsage());
}
GCHandle::GCSweepScope::~GCSweepScope() {
GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime());
handle_.heapUsageAfter(getUsage());
}
MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() {
return MemoryUsage{
mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(),
mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(),
};
}
GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) :
handle_(handle) {
handle_.extraObjectsUsageBefore(getUsage());
}
GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() {
GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime());
handle_.extraObjectsUsageAfter(getUsage());
}
GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
GCHandle::GCGlobalRootSetScope::~GCGlobalRootSetScope(){
handle_.globalRootSetCollected(globalRoots_, stableRoots_);
GCLogDebug(handle_.getEpoch(), "Collected global root set global=%" PRIu64 " stableRef=%" PRIu64 " in %" PRIu64" microseconds.",
globalRoots_, stableRoots_, getStageTime());
}
GCHandle::GCThreadRootSetScope::GCThreadRootSetScope(kotlin::gc::GCHandle& handle, mm::ThreadData& threadData) :
handle_(handle), threadData_(threadData) {}
GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){
handle_.threadRootSetCollected(threadData_, threadLocalRoots_, stackRoots_);
GCLogDebug(handle_.getEpoch(), "Collected root set for thread #%d: stack=%" PRIu64 " tls=%" PRIu64 " in %" PRIu64" microseconds.",
threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime());
}
GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){}
GCHandle::GCMarkScope::~GCMarkScope() {
handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes});
GCLogDebug(handle_.getEpoch(),
"Marked %" PRIu64 " objects in %" PRIu64 " microseconds in thread %d",
objectsCount, getStageTime(), konan::currentThreadId());
}
} // namespace kotlin::gc
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp
+121
View File
@@ -0,0 +1,121 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
#pragma once
#include <cstdint>
#include <pthread.h>
#include "Common.h"
#include "ThreadData.hpp"
#define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
#define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
namespace kotlin::gc {
class GCHandle;
struct MemoryUsage {
uint64_t objectsCount;
uint64_t totalObjectsSize;
};
class GCHandle {
class GCStageScopeUsTimer {
protected:
uint64_t startTime_ = konan::getTimeMicros();
uint64_t getStageTime() const { return (konan::getTimeMicros() - startTime_); }
};
class GCSweepScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
MemoryUsage getUsage();
public:
explicit GCSweepScope(GCHandle& handle);
~GCSweepScope();
};
class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
MemoryUsage getUsage();
public:
explicit GCSweepExtraObjectsScope(GCHandle& handle);
~GCSweepExtraObjectsScope();
};
class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
uint64_t globalRoots_ = 0;
uint64_t stableRoots_ = 0;
public:
explicit GCGlobalRootSetScope(GCHandle& handle);
~GCGlobalRootSetScope();
void addGlobalRoot() { globalRoots_++; }
void addStableRoot() { stableRoots_++; }
};
class GCThreadRootSetScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
mm::ThreadData& threadData_;
uint64_t stackRoots_ = 0;
uint64_t threadLocalRoots_ = 0;
public:
explicit GCThreadRootSetScope(GCHandle& handle, mm::ThreadData& threadData);
~GCThreadRootSetScope();
void addStackRoot() { stackRoots_++; }
void addThreadLocalRoot() { threadLocalRoots_++; }
};
class GCMarkScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
uint64_t objectsCount = 0;
uint64_t totalObjectSizeBytes = 0;
public:
explicit GCMarkScope(GCHandle& handle);
~GCMarkScope();
void addObject(uint64_t objectSize) {
totalObjectSizeBytes += objectSize;
objectsCount++;
}
};
uint64_t epoch_;
explicit GCHandle(uint64_t epoch) : epoch_(epoch) {}
void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences);
void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences);
void heapUsageBefore(MemoryUsage usage);
void heapUsageAfter(MemoryUsage usage);
void extraObjectsUsageBefore(MemoryUsage usage);
void extraObjectsUsageAfter(MemoryUsage usage);
void marked(MemoryUsage usage);
public:
static GCHandle create(uint64_t epoch);
static GCHandle createFakeForTests();
static GCHandle getByEpoch(uint64_t epoch);
static void ClearForTests();
uint64_t getEpoch() { return epoch_; }
void finished();
void finalizersDone();
void finalizersScheduled(uint64_t finalizersCount);
void suspensionRequested();
void threadsAreSuspended();
void threadsAreResumed();
GCSweepScope sweep() { return GCSweepScope(*this); }
GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); }
GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); }
GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); }
GCMarkScope mark() { return GCMarkScope(*this); }
MemoryUsage getMarked();
};
}
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+21 -28
View File
@@ -9,6 +9,7 @@
#include "ExtraObjectData.hpp"
#include "FinalizerHooks.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
#include "Logging.hpp"
#include "Memory.h"
#include "ObjectOps.hpp"
@@ -88,18 +89,15 @@ struct MarkStats {
};
template <typename Traits>
MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
MarkStats stats;
stats.rootSetSize = markQueue.size();
auto timeStart = konan::getTimeMicros();
void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept {
auto markHandle = handle.mark();
while (!Traits::isEmpty(markQueue)) {
ObjHeader* top = Traits::dequeue(markQueue);
RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top);
RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local());
stats.aliveHeapSet++;
stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
markHandle.addObject(mm::GetAllocatedHeapSize(top));
Traits::processInMark(markQueue, top);
@@ -112,14 +110,12 @@ MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
}
}
}
auto timeEnd = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds in thread %d", stats.aliveHeapSet, timeEnd - timeStart, konan::currentThreadId());
return stats;
}
template <typename Traits>
void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noexcept {
void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& objectFactory) noexcept {
objectFactory.ProcessDeletions();
auto sweepHandle = handle.sweepExtraObjects();
auto iter = objectFactory.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
auto &extraObject = *it;
@@ -140,8 +136,9 @@ void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noex
}
template <typename Traits>
typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
auto sweepHandle = handle.sweep();
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
if (Traits::TryResetMark(*it)) {
@@ -160,64 +157,60 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact
}
template <typename Traits>
typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory& objectFactory) noexcept {
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
auto iter = objectFactory.LockForIter();
return Sweep<Traits>(iter);
return Sweep<Traits>(handle, iter);
}
template <typename Traits>
void collectRootSetForThread(typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
auto handle = gcHandle.collectThreadRoots(thread);
thread.gc().OnStoppedForGC();
size_t stack = 0;
size_t tls = 0;
// TODO: Remove useless mm::ThreadRootSet abstraction.
for (auto value : mm::ThreadRootSet(thread)) {
if (internal::collectRoot<Traits>(markQueue, value.object)) {
switch (value.source) {
case mm::ThreadRootSet::Source::kStack:
++stack;
handle.addStackRoot();
break;
case mm::ThreadRootSet::Source::kTLS:
++tls;
handle.addThreadLocalRoot();
break;
}
}
}
RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
}
template <typename Traits>
void collectRootSetGlobals(typename Traits::MarkQueue& markQueue) noexcept {
void collectRootSetGlobals(GCHandle gcHandle, typename Traits::MarkQueue& markQueue) noexcept {
auto handle = gcHandle.collectGlobalRoots();
mm::StableRefRegistry::Instance().ProcessDeletions();
size_t global = 0;
size_t stableRef = 0;
// TODO: Remove useless mm::GlobalRootSet abstraction.
for (auto value : mm::GlobalRootSet()) {
if (internal::collectRoot<Traits>(markQueue, value.object)) {
switch (value.source) {
case mm::GlobalRootSet::Source::kGlobal:
++global;
handle.addGlobalRoot();
break;
case mm::GlobalRootSet::Source::kStableRef:
++stableRef;
handle.addStableRoot();
break;
}
}
}
RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
}
// TODO: This needs some tests now.
template <typename Traits, typename F>
void collectRootSet(typename Traits::MarkQueue& markQueue, F&& filter) noexcept {
void collectRootSet(GCHandle handle, typename Traits::MarkQueue& markQueue, F&& filter) noexcept {
Traits::clear(markQueue);
for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
if (!filter(thread))
continue;
thread.Publish();
collectRootSetForThread<Traits>(markQueue, thread);
collectRootSetForThread<Traits>(handle, markQueue, thread);
}
collectRootSetGlobals<Traits>(markQueue);
collectRootSetGlobals<Traits>(handle, markQueue);
}
} // namespace gc
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp
+10 -4
View File
@@ -174,13 +174,19 @@ public:
return testing::UnorderedElementsAreArray(objects);
}
gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
std_support::vector<ObjHeader*> objects;
for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader());
return gc::Mark<ScopedMarkTraits>(objects);
auto handle = gc::GCHandle::create(epoch_++);
gc::Mark<ScopedMarkTraits>(handle, objects);
handle.finished();
return handle.getMarked();
}
~MarkAndSweepUtilsMarkTest() { mm::GlobalData::Instance().gc().ClearForTests(); }
private:
uint64_t epoch_ = 0;
kotlin::ScopedMemoryInit memoryInit;
ScopedMarkTraits markTraits_;
};
@@ -196,8 +202,8 @@ size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<BaseObject>>
#define EXPECT_MARKED(stats, ...) \
do { \
std::initializer_list<std::reference_wrapper<BaseObject>> objects = {__VA_ARGS__}; \
EXPECT_THAT(stats.aliveHeapSet, objects.size()); \
EXPECT_THAT(stats.aliveHeapSetBytes, GetObjectsSize(objects)); \
EXPECT_THAT(stats.objectsCount, objects.size()); \
EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \
EXPECT_THAT(marked(), MarkedMatcher(objects)); \
} while (false)
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp
+2 -2
View File
@@ -199,8 +199,8 @@ public:
}
std_support::vector<ObjHeader*> Sweep() {
gc::SweepExtraObjects<SweepTraits>(extraObjectFactory_);
auto finalizers = gc::Sweep<SweepTraits>(objectFactory_);
gc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
auto finalizers = gc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
std_support::vector<ObjHeader*> objects;
for (auto node : finalizers.IterForTests()) {
objects.push_back(node.GetObjHeader());
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+16
View File
@@ -7,6 +7,8 @@
#include "GC.hpp"
#include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
using namespace kotlin;
@@ -61,12 +63,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
return impl_->objectFactory().GetObjectsCountUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
}
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
return impl_->gcScheduler().config();
}
void gc::GC::ClearForTests() noexcept {
impl_->objectFactory().ClearForTests();
GCHandle::ClearForTests();
}
void gc::GC::StartFinalizerThreadIfNeeded() noexcept {}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+16
View File
@@ -8,6 +8,8 @@
#include "GC.hpp"
#include "MarkAndSweepUtils.hpp"
#include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
using namespace kotlin;
@@ -74,12 +76,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
return impl_->objectFactory().GetObjectsCountUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
}
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
return impl_->gcScheduler().config();
}
void gc::GC::ClearForTests() noexcept {
impl_->objectFactory().ClearForTests();
GCHandle::ClearForTests();
}
void gc::GC::StartFinalizerThreadIfNeeded() noexcept {}
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+15 -45
View File
@@ -9,6 +9,7 @@
#include "CompilerConstants.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
#include "Logging.hpp"
#include "MarkAndSweepUtils.hpp"
#include "Memory.h"
@@ -103,7 +104,6 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
auto timeStartUs = konan::getTimeMicros();
bool didSuspend = mm::RequestThreadsSuspension();
if (!didSuspend) {
RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId());
@@ -111,8 +111,9 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
// TODO: This breaks if suspension is used by something apart from GC.
return false;
}
RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
gSafepointFlag = SafepointFlag::kNeedsSuspend;
auto gcHandle = GCHandle::create(epoch_++);
gcHandle.suspensionRequested();
mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
{
@@ -120,55 +121,27 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
ThreadStateGuard guard(ThreadState::kNative);
mm::WaitForThreadsSuspension();
auto timeSuspendUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
gcHandle.threadsAreSuspended();
auto& scheduler = gcScheduler_;
scheduler.gcData().OnPerformFullGC();
RuntimeLogInfo(
{kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
gc::collectRootSet<internal::MarkTraits>(markQueue_, [](mm::ThreadData&) { return true; });
auto timeRootSetUs = konan::getTimeMicros();
// Can be unsafe, because we've stopped the world.
auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [] (mm::ThreadData&) { return true; });
auto& extraObjectsDataFactory = mm::GlobalData::Instance().extraObjectDataFactory();
RuntimeLogInfo(
{kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(),
timeRootSetUs - timeSuspendUs);
auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
auto timeMarkUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs);
scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
auto timeSweepExtraObjectsUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
finalizerQueue = gc::Sweep<SweepTraits>(objectFactory_);
auto timeSweepUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs);
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
gc::SweepExtraObjects<SweepTraits>(gcHandle, extraObjectsDataFactory);
finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, objectFactory_);
kotlin::compactObjectPoolInMainThread();
// Can be unsafe, because we've stopped the world.
auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
gSafepointFlag = SafepointFlag::kNone;
mm::ResumeThreads();
auto timeResumeUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs);
auto finalizersCount = finalizerQueue.size();
auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
RuntimeLogInfo(
{kTagGC},
"Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
" microseconds",
epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs);
++epoch_;
lastGCTimestampUs_ = timeResumeUs;
gcHandle.threadsAreResumed();
gcHandle.finalizersScheduled(finalizerQueue.size());
gcHandle.finished();
}
// Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
@@ -177,11 +150,8 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
// TODO: These will actually need to be run on a separate thread.
AssertThreadState(ThreadState::kRunnable);
RuntimeLogDebug({kTagGC}, "Starting to run finalizers");
auto timeBeforeUs = konan::getTimeMicros();
finalizerQueue.Finalize();
auto timeAfterUs = konan::getTimeMicros();
RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs);
gcHandle.finalizersDone();
return true;
}
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+1 -2
View File
@@ -99,8 +99,7 @@ private:
// Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
bool PerformFullGC() noexcept;
size_t epoch_ = 0;
uint64_t lastGCTimestampUs_ = 0;
uint64_t epoch_ = 0;
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
GCScheduler& gcScheduler_;
kotlin-native/runtime/src/legacymm/cpp/Memory.cpp
+5
View File
@@ -3516,6 +3516,9 @@ void Kotlin_native_internal_GC_collect(KRef) {
#endif
}
extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) {
}
void Kotlin_native_internal_GC_collectCyclic(KRef) {
#if USE_CYCLIC_GC
if (g_hasCyclicCollector)
@@ -3862,3 +3865,5 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {}
bool kotlin::FinalizersThreadIsRunning() noexcept {
return false;
}
kotlin-native/runtime/src/main/cpp/Logging.cpp
+4 -14
View File
@@ -6,25 +6,15 @@
#include "Logging.hpp"
#include <array>
#if __has_include(<optional>)
#include <optional>
#elif __has_include(<experimental/optional>)
// TODO: Remove when wasm32 is gone.
#include <experimental/optional>
namespace std {
template <typename T>
using optional = std::experimental::optional<T>;
inline constexpr auto nullopt = std::experimental::nullopt;
} // namespace std
#else
#error "No <optional>"
#endif
#include "Format.h"
#include "KAssert.h"
#include "Porting.h"
#include "std_support/Map.hpp"
#include "std_support/String.hpp"
#include "std_support/Optional.hpp"
using namespace kotlin;
@@ -115,7 +105,7 @@ private:
class StderrLogger : public logging::internal::Logger {
public:
void Log(logging::Level level, std_support::span<const char* const> tags, std::string_view message) const noexcept override {
NO_EXTERNAL_CALLS_CHECK void Log(logging::Level level, std_support::span<const char* const> tags, std::string_view message) const noexcept override {
konan::consoleErrorUtf8(message.data(), message.size());
}
};
kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
#if __has_include(<optional>)
#include <optional>
#elif __has_include(<experimental/optional>)
// TODO: Remove when wasm32 is gone.
#include <experimental/optional>
namespace std {
template <typename T>
using optional = std::experimental::optional<T>;
inline constexpr auto nullopt = std::experimental::nullopt;
} // namespace std
#else
#error "No <optional>"
#endif
kotlin-native/runtime/src/main/cpp/std_support/README.md
+1
View File
@@ -25,3 +25,4 @@ Adjustments:
`std_support::kdelete` as a replacement for operator `delete` for objects created with custom `new`.
* `Deque.hpp`, `ForwardList.hpp`, `List.hpp`, `Map.hpp`, `Set.hpp`, `String.hpp`, `UnorderedMap.hpp`, `UnorderedSet.hpp`, `Vector.hpp` -
standard containers and `std_support::string` that default to using `std_support::allocator`.
* `Optional.hpp` - wrapper choosing correct way to include on different platforms.
kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp
+1
View File
@@ -53,6 +53,7 @@ public:
void ClearForTests() noexcept { extraObjects_.ClearForTests(); }
size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); }
size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); }
// requires LockForIter
void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); }
kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp
+24 -24
View File
@@ -145,6 +145,7 @@ public:
last_ = nodePtr;
++size_;
totalObjectsSizeBytes_ += Node::GetSizeForDataSize(dataSize);
RuntimeAssert(root_ != nullptr, "Must not be empty");
AssertCorrect();
return *nodePtr;
@@ -181,7 +182,9 @@ public:
owner_.last_ = last_;
last_ = nullptr;
owner_.size_ += size_;
owner_.totalObjectsSizeBytes_ += totalObjectsSizeBytes_;
size_ = 0;
totalObjectsSizeBytes_ = 0;
RuntimeAssert(root_ == nullptr, "Must be empty");
AssertCorrect();
@@ -216,6 +219,7 @@ public:
unique_ptr<Node> root_;
Node* last_ = nullptr;
size_t size_ = 0;
size_t totalObjectsSizeBytes_ = 0;
};
class Iterator {
@@ -348,13 +352,13 @@ public:
Iterator begin() noexcept { return Iterator(nullptr, owner_.root_.get()); }
Iterator end() noexcept { return Iterator(owner_.last_, nullptr); }
void EraseAndAdvance(Iterator& iterator) noexcept {
auto result = owner_.ExtractUnsafe(iterator.previousNode_);
void EraseAndAdvance(Iterator& iterator, size_t objectedSize) noexcept {
auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectedSize);
iterator.node_ = result.second;
}
void MoveAndAdvance(Consumer& consumer, Iterator& iterator) noexcept {
auto result = owner_.ExtractUnsafe(iterator.previousNode_);
void MoveAndAdvance(Consumer& consumer, Iterator& iterator, size_t objectSize) noexcept {
auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectSize);
iterator.node_ = result.second;
consumer.Insert(std::move(result.first));
}
@@ -373,40 +377,32 @@ public:
Iterable LockForIter() noexcept { return Iterable(*this); }
size_t GetSizeUnsafe() const noexcept { return size_; }
size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; }
void ClearForTests() {
root_.reset();
last_ = nullptr;
size_ = 0;
totalObjectsSizeBytes_ = 0;
}
private:
// Expects `mutex_` to be held by the current thread.
std::pair<unique_ptr<Node>, Node*> ExtractUnsafe(Node* previousNode) noexcept {
std::pair<unique_ptr<Node>, Node*> ExtractUnsafe(Node* previousNode, size_t objectSize) noexcept {
RuntimeAssert(root_ != nullptr, "Must not be empty");
AssertCorrectUnsafe();
if (previousNode == nullptr) {
// Extracting the root.
auto node = std::move(root_);
root_ = std::move(node->next_);
if (!root_) {
last_ = nullptr;
}
--size_;
AssertCorrectUnsafe();
return {std::move(node), root_.get()};
}
auto node = std::move(previousNode->next_);
previousNode->next_ = std::move(node->next_);
if (!previousNode->next_) {
unique_ptr<Node> &pointerToNext = (previousNode == nullptr) ? root_ : previousNode->next_;
auto node = std::move(pointerToNext);
pointerToNext = std::move(node->next_);
if (!pointerToNext) {
last_ = previousNode;
}
--size_;
totalObjectsSizeBytes_ -= objectSize;
AssertCorrectUnsafe();
return {std::move(node), previousNode->next_.get()};
return {std::move(node), pointerToNext.get()};
}
// Expects `mutex_` to be held by the current thread.
@@ -422,6 +418,7 @@ private:
unique_ptr<Node> root_;
Node* last_ = nullptr;
size_t size_ = 0;
size_t totalObjectsSizeBytes_ = 0;
SpinLock<MutexThreadStateHandling::kIgnore> mutex_;
};
@@ -656,10 +653,12 @@ public:
Iterator begin() noexcept { return Iterator(iter_.begin()); }
Iterator end() noexcept { return Iterator(iter_.end()); }
void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); }
void EraseAndAdvance(Iterator& iterator) noexcept {
iter_.EraseAndAdvance(iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader()));
}
void MoveAndAdvance(FinalizerQueue& queue, Iterator& iterator) noexcept {
iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_);
iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader()));
}
private:
@@ -672,7 +671,8 @@ public:
// Lock ObjectFactory for safe iteration.
Iterable LockForIter() noexcept { return Iterable(*this); }
size_t GetSizeUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); }
void ClearForTests() { storage_.ClearForTests(); }
kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp
+15 -15
View File
@@ -240,7 +240,7 @@ TEST(ObjectFactoryStorageTest, EraseFirst) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 1) {
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -268,7 +268,7 @@ TEST(ObjectFactoryStorageTest, EraseMiddle) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 2) {
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -296,7 +296,7 @@ TEST(ObjectFactoryStorageTest, EraseLast) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 3) {
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -323,7 +323,7 @@ TEST(ObjectFactoryStorageTest, EraseAll) {
{
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
}
}
@@ -345,7 +345,7 @@ TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) {
{
auto iter = storage.LockForIter();
auto it = iter.begin();
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
EXPECT_THAT(it, iter.end());
}
@@ -371,7 +371,7 @@ TEST(ObjectFactoryStorageTest, MoveFirst) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 1) {
iter.MoveAndAdvance(consumer, it);
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -403,7 +403,7 @@ TEST(ObjectFactoryStorageTest, MoveMiddle) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 2) {
iter.MoveAndAdvance(consumer, it);
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -435,7 +435,7 @@ TEST(ObjectFactoryStorageTest, MoveLast) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() == 3) {
iter.MoveAndAdvance(consumer, it);
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -466,7 +466,7 @@ TEST(ObjectFactoryStorageTest, MoveAll) {
{
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
iter.MoveAndAdvance(consumer, it);
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
}
}
@@ -499,7 +499,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() % 2 == 0) {
iter.MoveAndAdvance(consumer1, it);
iter.MoveAndAdvance(consumer1, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
@@ -508,7 +508,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) {
{
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
iter.MoveAndAdvance(consumer2, it);
iter.MoveAndAdvance(consumer2, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
}
}
@@ -577,7 +577,7 @@ TEST(ObjectFactoryStorageTest, MoveTheOnlyElement) {
{
auto iter = storage.LockForIter();
auto it = iter.begin();
iter.MoveAndAdvance(consumer, it);
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
EXPECT_THAT(it, iter.end());
}
@@ -612,8 +612,8 @@ TEST(ObjectFactoryStorageTest, MoveAndErase) {
auto iter = storage.LockForIter();
for (auto it = iter.begin(); it != iter.end();) {
++it;
iter.EraseAndAdvance(it);
iter.MoveAndAdvance(consumer, it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
}
}
@@ -759,7 +759,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
for (auto it = iter.begin(); it != iter.end();) {
if (it->Data<int>() % 2 != 0) {
iter.EraseAndAdvance(it);
iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
} else {
++it;
}
kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp
+1 -1
View File
@@ -81,7 +81,7 @@ NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::RequestThreadsSuspension() noexcept {
}
void kotlin::mm::WaitForThreadsSuspension() noexcept {
// Spin wating for threads to suspend. Ignore Native threads.
// Spin waiting for threads to suspend. Ignore Native threads.
while(!allThreads(isSuspendedOrNative)) {
yield();
}