azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Optimize intrusive_forward_list usage in GCs ^KT-54163

Browse Source 
* Add fallible push to intrusive_forward_list.
* Add fallible pop to intrusive_forward_list.
* Get rid of size_ member in intrusive_forward_list.
* Force all intrusive_forward_list items to have non-null next().
* Get rid of explicit colors in STMS and CMS.
* Do not queue weak reference for later processing in Mark.

Merge-request: KT-MR-7263
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>
This commit is contained in:
Alexander Shabalin
2022-10-06 20:16:53 +00:00
committed by Space Team
parent e7074e8a92
commit 5c9802ae84
11 changed files with 572 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+2 -4
View File
@@ -38,14 +38,12 @@ struct SweepTraits {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack;
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite) return false;
objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite);
return true;
return objectData.tryResetMark();
}
};
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+35 -41
View File
@@ -35,48 +35,39 @@ class FinalizerProcessor;
class ConcurrentMarkAndSweep : private Pinned {
public:
class ObjectData {
static inline constexpr unsigned colorMask = (1 << 1) - 1;
public:
enum class Color : unsigned {
kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
// All new objects are allocated with this color.
kBlack, // Objects encountered during mark phase.
};
Color color() const noexcept { return static_cast<Color>(getPointerBits(next_.load(std::memory_order_relaxed), colorMask)); }
void setColor(Color color) noexcept { next_.store(setPointerBits(clearPointerBits(next_.load(std::memory_order_relaxed), colorMask), static_cast<unsigned>(color)), std::memory_order_relaxed); }
bool atomicSetToBlack() noexcept {
ObjectData* before = next_.load(std::memory_order_relaxed);
if (getPointerBits(before, colorMask) != static_cast<unsigned>(Color::kWhite))
return false;
ObjectData* black = setPointerBits(before, static_cast<unsigned>(Color::kBlack));
bool success = next_.compare_exchange_strong(before, black, std::memory_order_relaxed);
RuntimeAssert(success || hasPointerBits(before, colorMask), "next_ must have been marked black");
return success;
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
ObjectData* next() const noexcept { return clearPointerBits(next_.load(std::memory_order_relaxed), colorMask); }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
auto bits = getPointerBits(next_.load(std::memory_order_relaxed), colorMask);
next_.store(setPointerBits(next, bits), std::memory_order_relaxed);
bool marked() const noexcept { return next() != nullptr; }
bool tryResetMark() noexcept {
if (next() == nullptr) return false;
next_.store(nullptr, std::memory_order_relaxed);
return true;
}
private:
// Color is encoded in low bits.
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_.load(std::memory_order_relaxed); }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_.store(next, std::memory_order_relaxed);
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
ObjectData* expected = nullptr;
return next_.compare_exchange_strong(expected, next, std::memory_order_relaxed);
}
std::atomic<ObjectData*> next_ = nullptr;
};
struct MarkQueueTraits {
static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
};
enum MarkingBehavior { kMarkOwnStack, kDoNotMark };
using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
using MarkQueue = intrusive_forward_list<ObjectData>;
class ThreadData : private Pinned {
public:
@@ -138,21 +129,24 @@ namespace internal {
struct MarkTraits {
using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue;
static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
static void clear(MarkQueue& queue) noexcept { queue.clear(); }
static ObjHeader* dequeue(MarkQueue& queue) noexcept {
auto& top = queue.front();
queue.pop_front();
auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(top);
return node->GetObjHeader();
static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
if (auto* top = queue.try_pop_front()) {
auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(*top);
return node->GetObjHeader();
}
return nullptr;
}
static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
if (!objectData.atomicSetToBlack()) return;
queue.push_front(objectData);
return queue.try_push_front(objectData);
}
static bool tryMark(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.tryMark();
}
static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+110 -111
View File
@@ -199,11 +199,9 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
return objects;
}
using Color = gc::ConcurrentMarkAndSweep::ObjectData::Color;
Color GetColor(ObjHeader* objHeader) {
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().color();
return nodeRef.ObjectData().marked();
}
WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -250,12 +248,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -263,12 +261,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
});
}
@@ -296,12 +294,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -309,12 +307,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
});
}
@@ -324,8 +322,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -339,8 +337,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -359,8 +357,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
@@ -376,14 +374,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(GetColor(weak1.header()), Color::kWhite);
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(weak1->referred, nullptr);
});
}
@@ -406,12 +404,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -419,12 +417,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -453,14 +451,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -468,12 +466,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -502,14 +500,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -519,12 +517,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -541,18 +539,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
});
}
@@ -581,14 +579,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
threadData.gc().ScheduleAndWaitFullGC();
@@ -597,12 +595,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -619,12 +617,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global2.header()), false);
});
}
@@ -638,14 +636,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(object.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(object.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
});
}
@@ -1114,13 +1112,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
auto& object1_local = AllocateObject(threadData);
object1 = &object1_local;
global1->field1 = object1_local.header();
while (weak.load() == nullptr);
while (weak.load() == nullptr)
;
threadData.gc().ScheduleAndWaitFullGC();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT((*weak.load())->referred, object1_local.header());
global1->field1 = nullptr;
@@ -1187,8 +1186,8 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
for (int i = 0; i < kDefaultThreadCount; ++i) {
gcFutures[i].wait();
// Verify that threads marked their own locals (but not their globals) according to the configured marking behavior:
ASSERT_THAT(GetColor(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack ? Color::kBlack : Color::kWhite);
ASSERT_THAT(GetColor(reachablesGlobals[i]), Color::kWhite);
ASSERT_THAT(IsMarked(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack);
ASSERT_THAT(IsMarked(reachablesGlobals[i]), false);
}
for (auto& future : gcFutures) {
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp
+3 -1
View File
@@ -23,6 +23,7 @@ struct MemoryUsage {
};
class GCHandle {
public:
class GCStageScopeUsTimer {
protected:
uint64_t startTime_ = konan::getTimeMicros();
@@ -86,6 +87,7 @@ class GCHandle {
}
};
private:
uint64_t epoch_;
explicit GCHandle(uint64_t epoch) : epoch_(epoch) {}
@@ -118,4 +120,4 @@ public:
MemoryUsage getMarked();
};
}
}
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+24 -29
View File
@@ -29,7 +29,7 @@ template <typename Traits>
void processFieldInMark(void* state, ObjHeader* field) noexcept {
auto& markQueue = *static_cast<typename Traits::MarkQueue*>(state);
if (field->heap()) {
Traits::enqueue(markQueue, field);
Traits::tryEnqueue(markQueue, field);
}
}
@@ -62,7 +62,7 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc
return false;
if (object->heap()) {
Traits::enqueue(markQueue, object);
Traits::tryEnqueue(markQueue, object);
} else {
// Each permanent and stack object has own entry in the root set, so it's okay to only process objects in heap.
Traits::processInMark(markQueue, object);
@@ -71,43 +71,38 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc
return true;
}
} // namespace internal
struct MarkStats {
// How many objects are alive.
size_t aliveHeapSet = 0;
// How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself.
size_t aliveHeapSetBytes = 0;
// How many roots are were marked.
size_t rootSetSize = 0;
void merge(MarkStats other) {
aliveHeapSet += other.aliveHeapSet;
aliveHeapSetBytes += other.aliveHeapSetBytes;
rootSetSize += other.rootSetSize;
// TODO: Consider making it noinline to keep loop in `Mark` small.
template <typename Traits>
void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQueue, mm::ExtraObjectData& extraObjectData, ObjHeader* object) noexcept {
if (auto weakCounter = extraObjectData.GetWeakReferenceCounter()) {
RuntimeAssert(
weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", object, weakCounter,
weakCounter->permanent(), weakCounter->local());
// Do not schedule WeakReferenceCounter but process it right away.
// This will skip markQueue interaction.
if (Traits::tryMark(weakCounter)) {
markHandle.addObject(mm::GetAllocatedHeapSize(weakCounter));
// WeakReferenceCounter is empty, but keeping this just in case.
Traits::processInMark(markQueue, weakCounter);
}
}
};
}
} // namespace internal
template <typename Traits>
void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept {
auto markHandle = handle.mark();
while (!Traits::isEmpty(markQueue)) {
ObjHeader* top = Traits::dequeue(markQueue);
RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top);
RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local());
while (ObjHeader* top = Traits::tryDequeue(markQueue)) {
// TODO: Consider moving it to the sweep phase to make this loop more tight.
// This, however, requires care with scheduler interoperation.
markHandle.addObject(mm::GetAllocatedHeapSize(top));
Traits::processInMark(markQueue, top);
// TODO: Consider moving it before processInMark to make the latter something of a tail call.
if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) {
if (auto weakCounter = extraObjectData->GetWeakReferenceCounter()) {
RuntimeAssert(
weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", top,
weakCounter, weakCounter->permanent(), weakCounter->local());
Traits::enqueue(markQueue, weakCounter);
}
internal::processExtraObjectData<Traits>(markHandle, markQueue, *extraObjectData, top);
}
}
}
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp
+10 -7
View File
@@ -124,25 +124,28 @@ public:
const std_support::unordered_set<ObjHeader*>& marked() const { return marked_; }
static bool isEmpty(const MarkQueue& queue) noexcept {
return queue.empty();
}
static void clear(MarkQueue& queue) noexcept {
queue.clear();
}
static ObjHeader* dequeue(MarkQueue& queue) noexcept {
static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
if (queue.empty()) return nullptr;
auto top = queue.back();
queue.pop_back();
return top;
}
static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto result = instance_->marked_.insert(object);
if (result.second) {
queue.push_back(object);
}
return result.second;
}
static bool tryMark(ObjHeader* object) noexcept {
auto result = instance_->marked_.insert(object);
return result.second;
}
static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
@@ -176,7 +179,7 @@ public:
gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
std_support::vector<ObjHeader*> objects;
for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader());
for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().GetObjHeader());
auto handle = gc::GCHandle::create(epoch_++);
gc::Mark<ScopedMarkTraits>(handle, objects);
handle.finished();
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+2 -4
View File
@@ -31,14 +31,12 @@ struct SweepTraits {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite) return false;
objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite);
return true;
return objectData.tryResetMark();
}
};
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+38 -33
View File
@@ -33,37 +33,40 @@ public:
};
class ObjectData {
static inline constexpr unsigned colorMask = (1 << 1) - 1;
public:
enum class Color : unsigned {
kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
// All new objects are allocated with this color.
kBlack, // Objects encountered during mark phase.
};
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
Color color() const noexcept { return static_cast<Color>(getPointerBits(next_, colorMask)); }
void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast<unsigned>(color)); }
bool marked() const noexcept { return next_ != nullptr; }
ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
auto bits = getPointerBits(next_, colorMask);
next_ = setPointerBits(next, bits);
bool tryResetMark() noexcept {
if (next_ == nullptr) return false;
next_ = nullptr;
return true;
}
private:
// Color is encoded in low bits.
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_; }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_ != nullptr) {
return false;
}
next_ = next;
return true;
}
ObjectData* next_ = nullptr;
};
struct MarkQueueTraits {
static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
};
using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
using MarkQueue = intrusive_forward_list<ObjectData>;
class ThreadData : private Pinned {
public:
@@ -112,22 +115,24 @@ namespace internal {
struct MarkTraits {
using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue;
static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
static void clear(MarkQueue& queue) noexcept { queue.clear(); }
static ObjHeader* dequeue(MarkQueue& queue) noexcept {
auto& top = queue.front();
queue.pop_front();
auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(top);
return node->GetObjHeader();
static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
if (auto* top = queue.try_pop_front()) {
auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(*top);
return node->GetObjHeader();
}
return nullptr;
}
static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
queue.push_front(objectData);
return queue.try_push_front(objectData);
}
static bool tryMark(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.tryMark();
}
static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+108 -109
View File
@@ -200,11 +200,9 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
return objects;
}
using Color = gc::SameThreadMarkAndSweep::ObjectData::Color;
Color GetColor(ObjHeader* objHeader) {
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().color();
return nodeRef.ObjectData().marked();
}
WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -246,12 +244,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -259,12 +257,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
});
}
@@ -292,12 +290,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -305,12 +303,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
});
}
@@ -320,8 +318,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -335,8 +333,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -355,8 +353,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
threadData.gc().ScheduleAndWaitFullGC();
@@ -372,14 +370,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(GetColor(weak1.header()), Color::kWhite);
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(weak1->referred, nullptr);
});
}
@@ -402,12 +400,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -415,12 +413,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -449,14 +447,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
@@ -464,12 +462,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -498,14 +496,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -515,12 +513,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -537,18 +535,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
});
}
@@ -577,14 +575,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(GetColor(global.header()), Color::kWhite);
ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
threadData.gc().ScheduleAndWaitFullGC();
@@ -593,12 +591,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
});
}
@@ -615,12 +613,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global2.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global2.header()), false);
});
}
@@ -634,14 +632,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(object.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(GetColor(global.header()), Color::kWhite);
EXPECT_THAT(GetColor(object.header()), Color::kWhite);
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
});
}
@@ -1098,13 +1096,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
auto& object1_local = AllocateObject(threadData);
object1 = &object1_local;
global1->field1 = object1_local.header();
while (weak.load() == nullptr);
while (weak.load() == nullptr)
;
threadData.gc().ScheduleAndWaitFullGC();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite);
ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite);
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT((*weak.load())->referred, object1_local.header());
global1->field1 = nullptr;
kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp
+156 -66
View File
@@ -11,29 +11,41 @@
#include "KAssert.h"
#include "Utils.hpp"
#include "std_support/Optional.hpp"
namespace kotlin {
template <typename T>
struct DefaultIntrusiveForwardListTraits {
static T* next(const T& value) noexcept { return value.next_; }
static T* next(const T& value) noexcept { return value.next(); }
static void setNext(T& value, T* next) noexcept { value.next_ = next; }
static void setNext(T& value, T* next) noexcept { value.setNext(next); }
static bool trySetNext(T& value, T* next) noexcept { return value.trySetNext(next); }
};
// Intrusive variant of `std::forward_list`. Notable differences:
// * The container does not own nodes. Care must be taken not to allow a node
// to be in two containers at once, or twice into the same container.
// * The container is move-only, and moving invalidates `before_begin` iterator.
// * insert_after, erase_after take `iterator` instead of `const_iterator`, because
// they do in fact require mutability via `Traits::setNext`.
// * When the node leaves the container, nothing clears `next` pointer inside it.
// Intrusive variant of `std::forward_list`.
//
// `Traits` must have 2 methods:
// static T* next(const T& value);
// static void setNext(T& value, T* next);
// NOTE: `setNext` and `next` must be callable even on uninitialized `T` (i.e. they
// should only access storage inside `T`).
// The container does not own nodes. The list structure is maintained by `T` itself via
// `Traits`. `Traits` must provide 3 operations:
//
// static T* next(const T& value); // obtain the next pointer
// static void setNext(T& value, T* next); // set the next pointer
// static bool trySetNext(T& value, T* next); // try to set the next pointer or return `false` if it's not possible. Used by `try_push_front`.
// The default `Traits` implementation expects `T` to provide all operations as member functions.
//
// Notable differences from regular containers:
// * To put `T` into different intrusive lists simultaneously it should
// provide custom `Traits` implementation to regulate which next pointer is used
// by which list.
// * It's not possible to keep the same `T` twice in the same list.
// * The container is move-only and moving invalidates `before_begin` iterator.
// * `insert_after`, `erase_after` take `iterator` instead of `const_iterator`,
// because they require mutability.
// * When the node is inserted into the container its next pointer is set
// to something non-null, but when its removed from the container nothing
// nulls the next pointer.
// * `Traits::trySetNext`, `Traits::setNext` and `Traits::next` must be callable even on uninitialized `T` (i.e. they should only access storage inside `T`).
template <typename T, typename Traits = DefaultIntrusiveForwardListTraits<T>>
class intrusive_forward_list : private MoveOnly {
public:
@@ -123,159 +135,237 @@ public:
pointer node_ = nullptr;
};
intrusive_forward_list() noexcept {
setNext(head(), nullptr);
}
// Complexity: O(1)
intrusive_forward_list() noexcept { clear(); }
intrusive_forward_list(intrusive_forward_list&& rhs) noexcept : size_(rhs.size_) {
// Complexity: O(1)
intrusive_forward_list(intrusive_forward_list&& rhs) noexcept {
// Since tail() is shared, there's no need to update the last node's next_.
setNext(head(), next(rhs.head()));
setNext(rhs.head(), nullptr);
rhs.size_ = 0;
rhs.clear();
}
// `InputIt` should dereference into `T&`.
// Complexity: O(last - first)
template <typename InputIt>
intrusive_forward_list(InputIt first, InputIt last) noexcept {
setNext(head(), nullptr);
intrusive_forward_list(InputIt first, InputIt last) noexcept : intrusive_forward_list() {
assign(std::move(first), std::move(last));
}
// Complexity: O(1)
~intrusive_forward_list() = default;
// Complexity: O(1)
intrusive_forward_list& operator=(intrusive_forward_list&& rhs) noexcept {
intrusive_forward_list tmp(std::move(rhs));
swap(tmp);
return *this;
}
// Complexity: O(1)
void swap(intrusive_forward_list& rhs) noexcept {
// Since tail() is shared, there's no need to swap the last nodes' next_.
using std::swap;
auto thisNext = next(head());
auto rhsNext = next(rhs.head());
swap(thisNext, rhsNext);
setNext(head(), thisNext);
setNext(rhs.head(), rhsNext);
swap(size_, rhs.size_);
}
// Rewrite the contents of `this` with nodes from range `[first, last)`.
// `InputIt` should dereference into `T&`.
// Complexity: O(last - first)
template <typename InputIt>
void assign(InputIt first, InputIt last) noexcept {
clear();
insert_after(before_begin(), std::move(first), std::move(last));
}
// Complexity: O(1)
reference front() noexcept { return *next(head()); }
// Complexity: O(1)
const_reference front() const noexcept { return *next(head()); }
// Iterator before the first node. Cannot be dereferenced.
// Complexity: O(1)
iterator before_begin() noexcept { return iterator(head()); }
// Iterator before the first node. Cannot be dereferenced.
// Complexity: O(1)
const_iterator before_begin() const noexcept { return const_iterator(head()); }
// Iterator before the first node. Cannot be dereferenced.
// Complexity: O(1)
const_iterator cbefore_begin() const noexcept { return const_iterator(head()); }
// Complexity: O(1)
iterator begin() noexcept { return iterator(next(head())); }
// Complexity: O(1)
const_iterator begin() const noexcept { return const_iterator(next(head())); }
// Complexity: O(1)
const_iterator cbegin() const noexcept { return const_iterator(next(head())); }
iterator end() noexcept { return iterator(); }
const_iterator end() const noexcept { return const_iterator(); }
const_iterator cend() const noexcept { return const_iterator(); }
// Complexity: O(1)
iterator end() noexcept { return iterator(tail()); }
// Complexity: O(1)
const_iterator end() const noexcept { return const_iterator(tail()); }
// Complexity: O(1)
const_iterator cend() const noexcept { return const_iterator(tail()); }
bool empty() const noexcept { return size_ == 0; }
// Complexity: O(1)
bool empty() const noexcept { return next(head()) == tail(); }
// Complexity: O(1)
size_type max_size() const noexcept { return std::numeric_limits<size_type>::max(); }
void clear() noexcept { setNext(head(), nullptr); size_ = 0; }
// Complexity: O(1)
void clear() noexcept { setNext(head(), tail()); }
// Insert `value` after `pos`. `pos` can be in range `[before_begin(), end())`.
// Returns iterator to the newly inserted element
// Complexity: O(1)
iterator insert_after(iterator pos, reference value) noexcept {
pointer nextNode = next(pos.node_);
RuntimeAssert(pos != end(), "Attempted to insert_after end()");
RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator");
setNext(&value, next(pos.node_));
setNext(pos.node_, &value);
setNext(&value, nextNode);
++size_;
return iterator(&value);
}
// Insert `[first, last)` after `pos`. `pos` can be in range `[before_begin(), end())`.
// `InputIt` should dereference into `T&`.
// Returns iterator to the last inserted element.
// Complexity: O(last - first)
template <typename InputIt>
iterator insert_after(iterator pos, InputIt first, InputIt last) noexcept {
RuntimeAssert(pos != end(), "Attempted to insert_after end()");
RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator");
pointer nextNode = next(pos.node_);
pointer prevNode = pos.node_;
size_t newSize = size_;
for (auto it = first; it != last; ++it) {
setNext(prevNode, &*it);
prevNode = &*it;
++newSize;
pointer newNode = &*it;
setNext(prevNode, newNode);
prevNode = newNode;
}
setNext(prevNode, nextNode);
size_ = newSize;
return iterator(prevNode);
}
// Erase a node after `pos`. `pos` can be in range `[begin(), end() - 1)`.
// This does not destroy the erased element, and it does not change its next pointer.
// Returns iterator to the next node of the erased one.
// Complexity: O(1)
iterator erase_after(iterator pos) noexcept {
pointer prevNode = pos.node_;
pointer nodeToErase = next(pos.node_);
if (!nodeToErase) {
return end();
}
pointer nextNode = next(nodeToErase);
setNext(prevNode, nextNode);
setNext(nodeToErase, nullptr);
--size_;
RuntimeAssert(pos != end(), "Attempted to erase_after end()");
RuntimeAssert(pos != iterator(), "Attempted to erase_after empty iterator");
RuntimeAssert(next(pos.node_) != tail(), "Attempted to erase_after the last node");
pointer nextNode = next(next(pos.node_));
setNext(pos.node_, nextNode);
return iterator(nextNode);
}
// Erase all nodes in range `(first, last)`.
// `first` can be in range `[before_begin(), last)`.
// `last` can be in range `(first, end()]`.
// This does not destroy erased elements, and it does not change their next pointers.
// Returns iterator to the next node of the last erased (i.e. returns `last`).
// Complexity: O(1)
iterator erase_after(iterator first, iterator last) noexcept {
size_ -= std::distance(first, last) - 1;
RuntimeAssert(first != end(), "Attempted to erase_after starting at end()");
RuntimeAssert(first != iterator(), "Attempted to erase_after starting at empty iterator");
RuntimeAssert(next(first.node_) != tail(), "Attempted to erase_after starting at the last node");
RuntimeAssert(last != iterator(), "Attempted to erase_after ending at empty iterator");
setNext(first.node_, last.node_);
return last;
}
// Insert a new node to the front.
// Equivalent to `insert_after(before_begin(), value)`.
// Complexity: O(1)
void push_front(reference value) noexcept { insert_after(before_begin(), value); }
// Try to insert a new node to the front.
// When setting the next node of `value` uses `Traits::trySetNext`.
// If `Traits::trySetNext` returns `true`, this operates like `push_front` and returns `true`.
// If `Traits::trySetNext` returns `false`, this doesn't change anything else and returns `false`.
// Complexity: O(1)
bool try_push_front(reference value) noexcept { return try_insert_after(before_begin(), value) != std::nullopt; }
// Erase a node at the front.
// This does not destroy the erased node and does not change its next pointer.
// Complexity: O(1)
void pop_front() noexcept { erase_after(before_begin()); }
void remove(reference value) noexcept {
// TODO: no need to move on after finding the first match.
return remove_if([&value](const_reference x) { return &x == &value; });
// Try to erase node at the front.
// If this list is empty, returns `nullptr`.
// Otherwise returns pointer to the node at the front and erases it.
// This does not destroy the erased node and does not change its next pointer.
// Complexity: O(1)
pointer try_pop_front() noexcept {
pointer top = next(head());
if (top == tail()) {
return nullptr;
}
setNext(head(), next(top));
return top;
}
// Erase node `value`.
// If the `value` is not in the list, does nothing.
// This does not destroy the erased node and does not change its next pointer.
// Complexity: O(n)
void remove(reference value) noexcept {
// TODO: no need to move on after finding the first match.
return remove_if([&value](const_reference x) noexcept { return &x == &value; });
}
// Erase all nodes satisfying predicate `P`.
// This does not destroy erased nodes and does not change their next pointer.
// Complexity: O(n)
template <typename P>
void remove_if(P p) noexcept {
size_t newSize = size_;
void remove_if(P p) noexcept(noexcept(p(std::declval<const_reference>()))) {
pointer prev = head();
pointer node = next(prev);
while (node) {
while (node != tail()) {
if (p(*node)) {
// The node is being removed.
node = next(node);
setNext(prev, node);
--newSize;
} else {
// The node is staying.
prev = node;
node = next(node);
}
}
size_ = newSize;
}
// TODO: Implement splice_after.
size_type size() const noexcept {
return size_;
}
private:
static pointer next(const_pointer node) noexcept { return Traits::next(*node); }
static void setNext(pointer node, pointer next) noexcept { return Traits::setNext(*node, next); }
pointer head() noexcept {
return reinterpret_cast<pointer>(headStorage_);
static bool trySetNext(pointer node, pointer next) noexcept { return Traits::trySetNext(*node, next); }
pointer head() noexcept { return reinterpret_cast<pointer>(headStorage_); }
const_pointer head() const noexcept { return reinterpret_cast<const_pointer>(headStorage_); }
static pointer tail() noexcept { return reinterpret_cast<pointer>(tailStorage_); }
// TODO: Consider making public.
std::optional<iterator> try_insert_after(iterator pos, reference value) noexcept {
RuntimeAssert(pos != end(), "Attempted to try_insert_after end()");
RuntimeAssert(pos != iterator(), "Attempted to try_insert_after empty iterator");
if (!trySetNext(&value, next(pos.node_))) {
return std::nullopt;
}
setNext(pos.node_, &value);
return iterator(&value);
}
const_pointer head() const noexcept {
return reinterpret_cast<const_pointer>(headStorage_);
}
alignas(value_type) char headStorage_[sizeof(value_type)] = { 0 };
size_t size_ = 0;
alignas(value_type) char headStorage_[sizeof(value_type)] = {0};
alignas(value_type) static inline char tailStorage_[sizeof(value_type)] = {0};
};
template <typename InputIt>
kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp
+84 -9
View File
@@ -45,9 +45,23 @@ public:
int& operator*() { return value_; }
const int& operator*() const { return value_; }
void clearNext() noexcept { next_ = nullptr; }
private:
friend struct DefaultIntrusiveForwardListTraits<Node>;
Node* next() const noexcept { return next_; }
void setNext(Node* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(Node* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_) return false;
next_ = next;
return true;
}
int value_;
// Use non-null marker to make sure inserting into the list properly updates this value.
Node* next_ = reinterpret_cast<Node*>(0x1);
@@ -68,16 +82,8 @@ MATCHER_P(isEmpty, expected, (expected == !negation) ? "is empty" : "is not empt
return expected == actual;
}
size_t getSize(const std::forward_list<Element>& list) {
return std::distance(list.begin(), list.end());
}
size_t getSize(const intrusive_forward_list<Node>& list) {
return list.size();
}
MATCHER_P(hasSize, expected, "") {
size_t actual = getSize(arg);
size_t actual = std::distance(arg.begin(), arg.end());
*result_listener << "of size " << actual;
return expected == actual;
}
@@ -622,3 +628,72 @@ TYPED_TEST(ForwardListTest, EraseAfterEmptyRangeFront) {
EXPECT_THAT(result, list.begin());
EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4);
}
TEST(InstrusiveForwardListTest, TryPushFrontSuccess) {
using List = intrusive_forward_list<Node>;
auto values = create<List>({1, 2, 3, 4});
List list(values.begin(), values.end());
typename List::value_type value(5);
value.clearNext();
auto result = list.try_push_front(value);
EXPECT_TRUE(result);
EXPECT_ELEMENTS_ARE(list, 5, 1, 2, 3, 4);
}
TEST(InstrusiveForwardListTest, TryPushFrontFailure) {
using List = intrusive_forward_list<Node>;
auto values = create<List>({1, 2, 3, 4});
List list(values.begin(), values.end());
typename List::value_type value(5);
auto result = list.try_push_front(value);
EXPECT_FALSE(result);
EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4);
}
TEST(InstrusiveForwardListTest, TryPushFrontEmptySuccess) {
using List = intrusive_forward_list<Node>;
List list;
typename List::value_type value(5);
value.clearNext();
auto result = list.try_push_front(value);
EXPECT_TRUE(result);
EXPECT_ELEMENTS_ARE(list, 5);
}
TEST(InstrusiveForwardListTest, TryPushFrontEmptyFailure) {
using List = intrusive_forward_list<Node>;
List list;
typename List::value_type value(5);
auto result = list.try_push_front(value);
EXPECT_FALSE(result);
EXPECT_ELEMENTS_ARE(list);
}
TEST(IntrusiveForwardListTest, TryPopFront) {
using List = intrusive_forward_list<Node>;
auto values = create<List>({1, 2, 3, 4});
List list(values.begin(), values.end());
auto& front = list.front();
auto result = list.try_pop_front();
EXPECT_THAT(result, &front);
EXPECT_ELEMENTS_ARE(list, 2, 3, 4);
}
TEST(IntrusiveForwardListTest, TryPopFrontIntoEmpty) {
using List = intrusive_forward_list<Node>;
auto values = create<List>({1});
List list(values.begin(), values.end());
auto& front = list.front();
auto result = list.try_pop_front();
EXPECT_THAT(result, &front);
EXPECT_ELEMENTS_ARE(list);
}
TEST(IntrusiveForwardListTest, TryPopFrontFromEmpty) {
using List = intrusive_forward_list<Node>;
auto values = create<List>({});
List list(values.begin(), values.end());
auto result = list.try_pop_front();
EXPECT_THAT(result, nullptr);
EXPECT_ELEMENTS_ARE(list);
}