azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Copy stms GC to start cms development

Browse Source
This commit is contained in:
Pavel Kunyavskiy
2021-11-23 11:46:29 +03:00
committed by Space
parent dd2849b7e7
commit 03299023f1
4 changed files with 1476 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+203
View File
@@ -0,0 +1,203 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "SameThreadMarkAndSweep.hpp"
#include <cinttypes>
#include "CompilerConstants.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "MarkAndSweepUtils.hpp"
#include "Memory.h"
#include "RootSet.hpp"
#include "Runtime.h"
#include "ThreadData.hpp"
#include "ThreadRegistry.hpp"
#include "ThreadSuspension.hpp"
using namespace kotlin;
namespace {
struct MarkTraits {
static bool IsMarked(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
}
static bool TryMark(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return false;
objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
return true;
};
};
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).GCObjectData();
return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.GCObjectData();
if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite) return false;
objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite);
return true;
}
};
struct FinalizeTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
};
// Global, because it's accessed on a hot path: avoid memory load from `this`.
std::atomic<gc::SameThreadMarkAndSweep::SafepointFlag> gSafepointFlag = gc::SameThreadMarkAndSweep::SafepointFlag::kNone;
} // namespace
ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept {
SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight);
}
ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight);
}
void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
threadData_.gcScheduler().OnSafePointAllocation(size);
SafepointFlag flag = gSafepointFlag.load();
if (flag != SafepointFlag::kNone) {
SafePointSlowPath(flag);
}
}
void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept {
auto didGC = gc_.PerformFullGC();
if (!didGC) {
// If we failed to suspend threads, someone else might be asking to suspend them.
threadData_.suspensionData().suspendIfRequested();
}
}
void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
PerformFullGC();
}
ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
threadData_.gcScheduler().OnSafePointRegular(weight);
SafepointFlag flag = gSafepointFlag.load();
if (flag != SafepointFlag::kNone) {
SafePointSlowPath(flag);
}
}
NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept {
RuntimeAssert(flag != SafepointFlag::kNone, "Must've been handled by the caller");
// No need to check for kNeedsSuspend, because `suspendIfRequested` checks for its own atomic.
threadData_.suspensionData().suspendIfRequested();
if (flag == SafepointFlag::kNeedsGC) {
RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint");
PerformFullGC();
}
}
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept {
mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() {
RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
gSafepointFlag = SafepointFlag::kNeedsGC;
});
}
bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
auto timeStartUs = konan::getTimeMicros();
bool didSuspend = mm::RequestThreadsSuspension();
if (!didSuspend) {
RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId());
// Somebody else suspended the threads, and so ran a GC.
// TODO: This breaks if suspension is used by something apart from GC.
return false;
}
RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
gSafepointFlag = SafepointFlag::kNeedsSuspend;
mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
{
// Switch state to native to simulate this thread being a GC thread.
ThreadStateGuard guard(ThreadState::kNative);
mm::WaitForThreadsSuspension();
auto timeSuspendUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
auto& scheduler = mm::GlobalData::Instance().gcScheduler();
scheduler.gcData().OnPerformFullGC();
RuntimeLogInfo(
{kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
auto graySet = collectRootSet();
auto timeRootSetUs = konan::getTimeMicros();
// Can be unsafe, because we've stopped the world.
auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
RuntimeLogInfo(
{kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(),
timeRootSetUs - timeSuspendUs);
auto markStats = gc::Mark<MarkTraits>(std::move(graySet));
auto timeMarkUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries);
scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
auto timeSweepExtraObjectsUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
auto timeSweepUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs);
// Can be unsafe, because we've stopped the world.
auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
gSafepointFlag = SafepointFlag::kNone;
mm::ResumeThreads();
auto timeResumeUs = konan::getTimeMicros();
RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs);
auto finalizersCount = finalizerQueue.size();
auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
RuntimeLogInfo(
{kTagGC},
"Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
" microseconds",
epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs);
++epoch_;
lastGCTimestampUs_ = timeResumeUs;
}
// Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
// try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing
// kotlin code.
// TODO: These will actually need to be run on a separate thread.
AssertThreadState(ThreadState::kRunnable);
RuntimeLogDebug({kTagGC}, "Starting to run finalizers");
auto timeBeforeUs = konan::getTimeMicros();
finalizerQueue.Finalize();
auto timeAfterUs = konan::getTimeMicros();
RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs);
return true;
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+89
View File
@@ -0,0 +1,89 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <cstddef>
#include "Allocator.hpp"
#include "GCScheduler.hpp"
#include "ObjectFactory.hpp"
#include "Types.h"
#include "Utils.hpp"
namespace kotlin {
namespace mm {
class ThreadData;
}
namespace gc {
// Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads.
class SameThreadMarkAndSweep : private Pinned {
public:
enum class SafepointFlag {
kNone,
kNeedsSuspend,
kNeedsGC,
};
class ObjectData {
public:
enum class Color {
kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
// All new objects are allocated with this color.
kBlack, // Objects encountered during mark phase.
};
Color color() const noexcept { return color_; }
void setColor(Color color) noexcept { color_ = color; }
private:
Color color_ = Color::kWhite;
};
class ThreadData : private Pinned {
public:
using ObjectData = SameThreadMarkAndSweep::ObjectData;
using Allocator = AllocatorWithGC<AlignedAllocator, ThreadData>;
explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
~ThreadData() = default;
void SafePointFunctionPrologue() noexcept;
void SafePointLoopBody() noexcept;
void SafePointExceptionUnwind() noexcept;
void SafePointAllocation(size_t size) noexcept;
void PerformFullGC() noexcept;
void OnOOM(size_t size) noexcept;
Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); }
private:
void SafePointRegular(size_t weight) noexcept;
void SafePointSlowPath(SafepointFlag flag) noexcept;
SameThreadMarkAndSweep& gc_;
mm::ThreadData& threadData_;
};
using Allocator = ThreadData::Allocator;
SameThreadMarkAndSweep() noexcept;
~SameThreadMarkAndSweep() = default;
private:
// Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
bool PerformFullGC() noexcept;
size_t epoch_ = 0;
uint64_t lastGCTimestampUs_ = 0;
};
} // namespace gc
} // namespace kotlin
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+1165
View File
File diff suppressed because it is too large Load Diff
kotlin-native/runtime/src/gc/cms/cpp/GC.hpp
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include "SameThreadMarkAndSweep.hpp"
namespace kotlin {
namespace gc {
using GC = kotlin::gc::SameThreadMarkAndSweep;
inline constexpr bool kSupportsMultipleMutators = true;
} // namespace gc
} // namespace kotlin