diff --git a/kotlin-native/runtime/src/custom_alloc/README.md b/kotlin-native/runtime/src/custom_alloc/README.md new file mode 100644 index 00000000000..0b0e323eeb3 --- /dev/null +++ b/kotlin-native/runtime/src/custom_alloc/README.md @@ -0,0 +1,357 @@ +# Overview of the allocator + +This document describes the internals of the custom allocator. The presentation +here is not fully true to the implementation, as the design is still work in +progress. + +The main idea of the custom allocator is to divide system memory into chunks +(pages) that each can be swept independently, in memory consecutive order. +Every allocation ends up as a block of memory inside a page. Each page keeps +track of the size of each block it contains; how this is done depends on the +page type, with different page types optimized for different allocation sizes. +All the memory blocks are consecutive within a page, so the size of a block +also tells where the next block begins. Paired with an additional mechanism +(page type dependent) for determining whether a block is allocated, we can +iterate through the allocated blocks. + +When a thread allocates memory for an object, it will find a page suitable for +the given allocation size. Each thread holds on to a number of pages for the +different size categories. The typical case is that the thread’s current page +for the given size can fit the requested allocation. If that is not the case, a +different page for that size category is requested from the shared allocation +space. The requested page can either be readily available (already prepared by +the GC thread), or it might need to be swept first, or it might be newly +created. + +The GC thread has a new responsibility when using this allocator: While the +mutator threads are paused at the start of GC, the GC thread must prepare the +allocator for sweeping. This does two things. First, it marks all pages as +“needs to be swept before next use”. Second, it releases pages that threads are +holding on to, by clearing the thread local variables for each thread. + +It is possible to have several independent allocation spaces at the same time. +This is currently useful for testing, but is potentially useful in other +settings. + +# Detailed Design + +All allocations are made through a `CustomAllocator` object. + +## [CustomAllocator](cpp/CustomAllocator.hpp) + +```cpp +class CustomAllocator { +public: + CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler); + ObjectHeader* CreateObject(TypeInfo* type); + ArrayHeader* CreateArray(TypeInfo* type, uint32_t count); + ExtraObjectData* CreateExtraObject(); + void PrepareForGc(); + +private: + uint8_t* Allocate(uint64_t cellCount); + uint8_t* AllocateInSingleObjectPage(uint64_t cellCount); + uint8_t* AllocateInNextFitPage(uint32_t cellCount); + uint8_t* AllocateInFixedBlockPage(uint32_t cellCount); + + Heap& heap_; + GCSchedulerThreadData& gcScheduler_; + NextFitPage* nextFitPage_; + FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE]; + ExtraObjectPage* extraObjectPage_; +}; +``` + +The primary responsibility of this class is to delegate each requested +allocation to pages of the appropriate type, based on allocation size. To do +this, it requests pages from the shared allocation space (`Heap`) and stores +pages for later allocations. Each thread thus owns a number of pages for +different allocation sizes, but at most one for each size class. When +allocating, the `CustomAllocator` will first try to allocate in one of its +owned pages. If this fails, it will request a new page for that size class from +a shared `Heap` object. `SingleObjectPages` are never kept by the +`CustomAllocator`, since they are created specifically for a single allocation, +with no extra space. + +## [Heap](cpp/Heap.hpp) + +```cpp +class Heap { +public: + void PrepareForGC(); + + void Sweep(); + + AtomicStack SweepExtraObjects(GCHandle gcHandle); + + FixedBlockPage* GetFixedBlockPage(uint32_t cellCount); + NextFitPage* GetNextFitPage(uint32_t cellCount); + SingleObjectPage* GetSingleObjectPage(uint64_t cellCount); + ExtraObjectPage* GetExtraObjectPage(); + +private: + PageStore fixedBlockPages_[MAX_BLOCK_SIZE]; + PageStore nextFitPages_; + PageStore singleObjectPages_; + AtomicStack extraObjectPages_; + AtomicStack usedExtraObjectPages_; +}; +``` + +A `Heap` object represents a shared allocation space for multiple +`CustomAllocator`s, which can request pages through one of the +`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also +provides a method for sweeping through all blocks that have been allocated in +this heap. The `Heap` object is the synchronization point, and guarantees that +every page is returned at most once. Page ownership is thus implicitly given to +the thread that called the method. The `Heap` object keeps track of all pages, +so there is no need to explicitly return ownership of a page. Internally, a +`Heap` keeps the pages for each size class in a `PageStore`. This means one for +`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes +for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two +`AtomicStack`s, since they require different handling during sweeping. + +## [PageStore](cpp/PageStore.hpp) + +```cpp +template +class PageStore { +public: + void PrepareForGC(); + void Sweep(); + void SweepAndFree(); + PageType* GetPage(uint32_t cellCount); + PageType* NewPage(uint64_t cellCount); + +private: + AtomicStack empty_; + AtomicStack ready_; + AtomicStack used_; + AtomicStack unswept_; +}; +``` + +A PageStore is responsible for keeping track of all pages of a given type and +size class. Each of the pages are in one of four stacks. The stack, that a +given page is in, determines its current state: + +* `unswept_`: have not yet been swept since the last GC cycle. +* `ready_`: are ready for allocation; has been swept by the GC thread. +* `used_`: has been given to some thread for allocation; it might still be used + for allocation, or it might have been discarded with not enough space left. + Will not be used until the next GC cycle. +* `empty_`: same as `ready_`, but does not contain any objects. Will be freed + before the next GC, if not needed before then. + +When a page is requested, the page is taken from `ready_`, if there are any. +Otherwise, an `unswept_` page is taken and swept before returning. If there are +no unswept pages either, an empty page is taken, if there are any. Otherwise a +new page is created in the size category. All returned pages are moved to +`used_`. During the marking phase, all remaining pages in `empty_` are freed, +and all other pages are moved to `unswept_`. The GC thread will go through all +`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one +of the other threads sweeps a page from `unswept_`, it is moved directly to +`used_`, as it is claimed by the `CustomAllocator` that swept it. + +`SingleObjectPage`s are treated slightly differently, because they are created +for one specific single allocation, and not reused when that allocation is +freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without +checking any of the stacks, and during sweeping, they are freed directly rather +than being put into the `empty_` stack. Ideally, there would only be two stacks +in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little +is lost by just using the existing `PageStore` logic used for the other pages. + +## [AtomicStack](cpp/AtomicStack.hpp) + +```cpp +template +class AtomicStack { +public: + PageType* Pop(); + void Push(PageType* elm); + void TransferAllFrom(AtomicStack& src); + bool isEmpty(); + + +private: + std::atomic stack_; +}; +``` + +The only place where atomics are used are in the stacks inside the `PageStore`. +All page classes have a non-atomic next pointer, to be used for linking up in +exactly one stack. `Pop` and `Push` are implemented with compare-and-swap +operations. The class is thread safe, except for if an element is freed while +another thread tries to Pop it from a stack. + +# Page types + +This section is likely to change, given the likely introduction of additional +page types. It also describes some details about which page type is chosen for +a given allocation, which is also likely to change. + +There are four different page types, but they all share the feature that they +can be swept independently. The Sweep methods return whether there were any +live objects in the page after sweeping. If not, the page will be given back to +the OS. + +## [FixedBlockPage](cpp/FixedBlockPage.hpp) + +```cpp +class FixedBlockPage { +public: + FixedBlockPage(uint32_t blockSize); + uint8_t* TryAllocate(); + bool Sweep() noexcept; + +private: + FixedBlockPage* next_; // used by AtomicStack + uint32_t blockSize_; + FixedBlockCell* nextFree_; + FixedBlockCell cells_[]; +}; +``` + +All sufficiently small allocations (currently arbitrary <1KiB) are directed to +a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations +are expected to be in this page type. A `FixedBlockPage` has a singly-linked +free-list of all free blocks. Allocating always happens in the first free block +in the page. + +```cpp +struct FixedBlockCell { + union { + uint8_t data[]; + FixedBlockCell* nextFree; + } +}; +``` + +The important point is that all links in the list point forward in the page, so +all blocks between two consecutive links are implicitly allocated. Sweeping a +`FixedBlockPage` consists of walking the free-list forward, and sweeping all +blocks in between the links, maintaining the free list when blocks are freed. + +Each small page takes up the same amount of space, independent of block size, +so larger block size implies fewer blocks per page. This size is arbitrarily +chosen to be 64 KiB, but this might change. + +## [NextFitPage](cpp/NextFitPage.hpp) + +```cpp +class NextFitPage { +public: + NextFitPage(uint32_t cellCount); + Cell* TryAllocate(uint32_t cellCount); + bool Sweep(); + +private: + NextFitPage* next_; // used by AtomicStack + Cell* curBlock_; + Cell cells_[]; +}; +``` + +Allocations that could theoretically fit in a `FixedBlockPage`, but would +require too large a block size (arbitrary >=1KiB), are allocated in a +`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64 +KiB for experiments). All blocks in a `NextFitPage` have a header that tells how +big the block is, and whether it is allocated or not. There are no gaps between +blocks, so the size of a block also tells where the next block is. The header +information fits inside a 8 byte `Cell`. + +```cpp +class Cell { +public: + Cell(uint32_t size); + uint8_t* TryAllocate(uint32_t cellCount); + +private: + uint32_t isAllocated_; + uint32_t size_; + uint8_t data_[]; +}; +``` + +The page keeps a reference to a currently active block, and will try to bump +allocate inside that block. If allocation does not fit, we move to the next +block that fits. If no block in the page fits the requested size, the page is +abandoned until the next GC. + +## [SingleObjectPage](cpp/SingleObjectPage.hpp) + +```cpp +class SingleObjectPage { +public: + SingleObjectPage(uint64_t cellCount); + bool Sweep(); + +private: + SingleObjectPage* next_; // used by AtomicStack +}; +``` + +Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`, +which only contains that single block of the requested size. They are also +handled slightly differently by both `Heap` and `CustomAllocator`. First off, +`Heap::GetSingleObjectPage` will never check existing pages, and instead just +allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to +any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC +thread. + +## [ExtraObjectPage](cpp/ExtraObjectPage.hpp) + +```cpp +class ExtraObjectPage { +public: + ExtraObjectPage(); + ExtraObjectData* TryAllocate(); + bool Sweep(FinalizerQueue& queue); + + +private: + ExtraObjectPage* next_; // used by AtomicStack + ExtraObjectCell* nextFree_; + ExtraObjectCell cells_[]; +}; +``` + +Extra objects are used for attaching additional data to some objects. This is +used for objects that require special handling during garbage collection: + +* objects with finalizers +* weak references +* interop references + +Extra objects are allocated in `ExtraObjectPage`s, which are very similar to +`FixedBlockPage`s. They primarily differ in how they are swept, since it is +during sweeping of `ExtraObject`s that scheduling of finalization happens. If +an object that requires finalization is found, it is added to the +`FinalizerQueue` given as argument. The cells are also slightly different, in +that they add a new field that allows the cells to be added to the finalizer +queue. + +```cpp +struct ExtraObjectCell { + ExtraObjectCell* next_; // used by AtomicStack + ExtraObjectData data_; +}; +``` + +# Finalizers + +Section like to change. + +In the existing memory model, finalization tasks are found and scheduled during +sweeping of regular objects. The objects to be finalized are chained together +using a pointer in the Node header, added to all allocated objects. This header +is not needed in the custom allocator, apart from linking in the finalization +queue. + +We therefore reintroduce this pointer in a header for `ExtraObjectData`. For +this, we reuse the `ExtraObjectCell` as header for both free list pointer and as +linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`. + +# Enabling the allocator + +The custom allocator is enabled with the compiler flag -Xallocator=custom. diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp index b37aa9f5352..81f7141247e 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp @@ -27,7 +27,7 @@ public: Cell* Next() noexcept; private: - friend class MediumPage; + friend class NextFitPage; uint32_t isAllocated_; uint32_t size_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp index a046f46f2b9..6fc90e7b851 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp @@ -9,23 +9,23 @@ #include #include -#include "SmallPage.hpp" -#include "MediumPage.hpp" +#include "FixedBlockPage.hpp" +#include "NextFitPage.hpp" #include "ExtraObjectPage.hpp" inline constexpr const size_t KiB = 1024; -inline constexpr const size_t SMALL_PAGE_SIZE = (256 * KiB); -inline constexpr const int SMALL_PAGE_MAX_BLOCK_SIZE = 128; -inline constexpr const size_t SMALL_PAGE_CELL_COUNT = - ((SMALL_PAGE_SIZE - sizeof(kotlin::alloc::SmallPage)) / sizeof(kotlin::alloc::SmallCell)); +inline constexpr const size_t FIXED_BLOCK_PAGE_SIZE = (256 * KiB); +inline constexpr const int FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE = 128; +inline constexpr const size_t FIXED_BLOCK_PAGE_CELL_COUNT = + ((FIXED_BLOCK_PAGE_SIZE - sizeof(kotlin::alloc::FixedBlockPage)) / sizeof(kotlin::alloc::FixedBlockCell)); -inline constexpr const size_t MEDIUM_PAGE_SIZE = (256 * KiB); -inline constexpr const size_t MEDIUM_PAGE_CELL_COUNT = - ((MEDIUM_PAGE_SIZE - sizeof(kotlin::alloc::MediumPage)) / sizeof(kotlin::alloc::Cell)); +inline constexpr const size_t NEXT_FIT_PAGE_SIZE = (256 * KiB); +inline constexpr const size_t NEXT_FIT_PAGE_CELL_COUNT = + ((NEXT_FIT_PAGE_SIZE - sizeof(kotlin::alloc::NextFitPage)) / sizeof(kotlin::alloc::Cell)); -// MEDIUM_PAGE_CELL_COUNT minus one cell for header minus another for the 0-sized dummy block at cells_[0] -inline constexpr const size_t MEDIUM_PAGE_MAX_BLOCK_SIZE = (MEDIUM_PAGE_CELL_COUNT - 2); +// NEXT_FIT_PAGE_CELL_COUNT minus one cell for header minus another for the 0-sized dummy block at cells_[0] +inline constexpr const size_t NEXT_FIT_PAGE_MAX_BLOCK_SIZE = (NEXT_FIT_PAGE_CELL_COUNT - 2); inline constexpr const size_t EXTRA_OBJECT_PAGE_SIZE = 64 * KiB; inline constexpr const int EXTRA_OBJECT_COUNT = diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 49e9a7e971f..a5dfedc0571 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -17,13 +17,12 @@ #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" #include "GCScheduler.hpp" -#include "LargePage.hpp" -#include "MediumPage.hpp" +#include "SingleObjectPage.hpp" +#include "NextFitPage.hpp" #include "Memory.h" -#include "SmallPage.hpp" +#include "FixedBlockPage.hpp" #include "GCImpl.hpp" #include "TypeInfo.h" -#include "Types.h" namespace kotlin::alloc { @@ -55,9 +54,9 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce } CustomAllocator::CustomAllocator(Heap& heap, gc::GCSchedulerThreadData& gcScheduler) noexcept : - heap_(heap), gcScheduler_(gcScheduler), mediumPage_(nullptr), extraObjectPage_(nullptr) { + heap_(heap), gcScheduler_(gcScheduler), nextFitPage_(nullptr), extraObjectPage_(nullptr) { CustomAllocInfo("CustomAllocator::CustomAllocator(heap)"); - memset(smallPages_, 0, sizeof(smallPages_)); + memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); } ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { @@ -115,8 +114,8 @@ mm::ExtraObjectData& CustomAllocator::CreateExtraObjectDataForObject( void CustomAllocator::PrepareForGC() noexcept { CustomAllocInfo("CustomAllocator@%p::PrepareForGC()", this); - mediumPage_ = nullptr; - memset(smallPages_, 0, sizeof(smallPages_)); + nextFitPage_ = nullptr; + memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); extraObjectPage_ = nullptr; } @@ -125,49 +124,49 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size); uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell); uint8_t* ptr; - if (cellCount <= SMALL_PAGE_MAX_BLOCK_SIZE) { - ptr = AllocateInSmallPage(cellCount); - } else if (cellCount > MEDIUM_PAGE_MAX_BLOCK_SIZE) { - ptr = AllocateInLargePage(cellCount); + if (cellCount <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE) { + ptr = AllocateInFixedBlockPage(cellCount); + } else if (cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE) { + ptr = AllocateInSingleObjectPage(cellCount); } else { - ptr = AllocateInMediumPage(cellCount); + ptr = AllocateInNextFitPage(cellCount); } memset(ptr, 0, size); return ptr; } -uint8_t* CustomAllocator::AllocateInLargePage(uint64_t cellCount) noexcept { - CustomAllocDebug("CustomAllocator::AllocateInLargePage(%" PRIu64 ")", cellCount); - uint8_t* block = heap_.GetLargePage(cellCount)->TryAllocate(); +uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept { + CustomAllocDebug("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount); + uint8_t* block = heap_.GetSingleObjectPage(cellCount)->TryAllocate(); return block; } -uint8_t* CustomAllocator::AllocateInMediumPage(uint32_t cellCount) noexcept { - CustomAllocDebug("CustomAllocator::AllocateInMediumPage(%u)", cellCount); - if (mediumPage_) { - uint8_t* block = mediumPage_->TryAllocate(cellCount); +uint8_t* CustomAllocator::AllocateInNextFitPage(uint32_t cellCount) noexcept { + CustomAllocDebug("CustomAllocator::AllocateInNextFitPage(%u)", cellCount); + if (nextFitPage_) { + uint8_t* block = nextFitPage_->TryAllocate(cellCount); if (block) return block; } CustomAllocDebug("Failed to allocate in curPage"); while (true) { - mediumPage_ = heap_.GetMediumPage(cellCount); - uint8_t* block = mediumPage_->TryAllocate(cellCount); + nextFitPage_ = heap_.GetNextFitPage(cellCount); + uint8_t* block = nextFitPage_->TryAllocate(cellCount); if (block) return block; } } -uint8_t* CustomAllocator::AllocateInSmallPage(uint32_t cellCount) noexcept { - CustomAllocDebug("CustomAllocator::AllocateInSmallPage(%u)", cellCount); - SmallPage* page = smallPages_[cellCount]; +uint8_t* CustomAllocator::AllocateInFixedBlockPage(uint32_t cellCount) noexcept { + CustomAllocDebug("CustomAllocator::AllocateInFixedBlockPage(%u)", cellCount); + FixedBlockPage* page = fixedBlockPages_[cellCount]; if (page) { uint8_t* block = page->TryAllocate(); if (block) return block; } - CustomAllocDebug("Failed to allocate in current SmallPage"); - while ((page = heap_.GetSmallPage(cellCount))) { + CustomAllocDebug("Failed to allocate in current FixedBlockPage"); + while ((page = heap_.GetFixedBlockPage(cellCount))) { uint8_t* block = page->TryAllocate(); if (block) { - smallPages_[cellCount] = page; + fixedBlockPages_[cellCount] = page; return block; } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index d9f74e46b14..5b058ca013a 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -13,9 +13,9 @@ #include "ExtraObjectPage.hpp" #include "GCScheduler.hpp" #include "Heap.hpp" -#include "MediumPage.hpp" +#include "NextFitPage.hpp" #include "Memory.h" -#include "SmallPage.hpp" +#include "FixedBlockPage.hpp" namespace kotlin::alloc { @@ -36,14 +36,14 @@ public: private: uint8_t* Allocate(uint64_t cellCount) noexcept; - uint8_t* AllocateInLargePage(uint64_t cellCount) noexcept; - uint8_t* AllocateInMediumPage(uint32_t cellCount) noexcept; - uint8_t* AllocateInSmallPage(uint32_t cellCount) noexcept; + uint8_t* AllocateInSingleObjectPage(uint64_t cellCount) noexcept; + uint8_t* AllocateInNextFitPage(uint32_t cellCount) noexcept; + uint8_t* AllocateInFixedBlockPage(uint32_t cellCount) noexcept; Heap& heap_; gc::GCSchedulerThreadData& gcScheduler_; - MediumPage* mediumPage_; - SmallPage* smallPages_[SMALL_PAGE_MAX_BLOCK_SIZE + 1]; + NextFitPage* nextFitPage_; + FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; ExtraObjectPage* extraObjectPage_; }; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp index 48378da66c2..c9e402e0996 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp @@ -39,9 +39,9 @@ TEST(CustomAllocTest, SmallAllocNonNull) { } } -TEST(CustomAllocTest, SmallAllocSameSmallPage) { - const int N = SMALL_PAGE_CELL_COUNT / SMALL_PAGE_MAX_BLOCK_SIZE; - for (int blocks = MIN_BLOCK_SIZE; blocks < SMALL_PAGE_MAX_BLOCK_SIZE; ++blocks) { +TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { + const int N = FIXED_BLOCK_PAGE_CELL_COUNT / FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; + for (int blocks = MIN_BLOCK_SIZE; blocks < FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blocks) { Heap heap; kotlin::gc::GCSchedulerConfig config; kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); @@ -51,31 +51,31 @@ TEST(CustomAllocTest, SmallAllocSameSmallPage) { for (int i = 1; i < N; ++i) { uint8_t* obj = reinterpret_cast(ca.CreateObject(&fakeType)); uint64_t dist = abs(obj - first); - EXPECT_TRUE(dist < SMALL_PAGE_SIZE); + EXPECT_TRUE(dist < FIXED_BLOCK_PAGE_SIZE); } } } -TEST(CustomAllocTest, SmallPageThreshold) { +TEST(CustomAllocTest, FixedBlockPageThreshold) { Heap heap; kotlin::gc::GCSchedulerConfig config; kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); - const int FROM = SMALL_PAGE_MAX_BLOCK_SIZE - 10; - const int TO = SMALL_PAGE_MAX_BLOCK_SIZE + 10; + const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10; + const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; ca.CreateObject(&fakeType); } } -TEST(CustomAllocTest, MediumPageThreshold) { +TEST(CustomAllocTest, NextFitPageThreshold) { Heap heap; kotlin::gc::GCSchedulerConfig config; kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); - const int FROM = MEDIUM_PAGE_MAX_BLOCK_SIZE - 10; - const int TO = MEDIUM_PAGE_MAX_BLOCK_SIZE + 10; + const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10; + const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; ca.CreateObject(&fakeType); @@ -94,7 +94,7 @@ TEST(CustomAllocTest, TwoAllocatorsDifferentPages) { uint8_t* obj1 = reinterpret_cast(ca1.CreateObject(&fakeType)); uint8_t* obj2 = reinterpret_cast(ca2.CreateObject(&fakeType)); uint64_t dist = abs(obj2 - obj1); - EXPECT_TRUE(dist >= SMALL_PAGE_SIZE); + EXPECT_TRUE(dist >= FIXED_BLOCK_PAGE_SIZE); } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp index 4f8208838ff..6e0fb16a5d5 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp @@ -11,7 +11,6 @@ #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" #include "gtest/gtest.h" -#include "TypeInfo.h" namespace { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp new file mode 100644 index 00000000000..2fae4a792a5 --- /dev/null +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -0,0 +1,76 @@ +/* + * Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "FixedBlockPage.hpp" + +#include +#include +#include +#include + +#include "CustomLogging.hpp" +#include "CustomAllocConstants.hpp" +#include "GCApi.hpp" + +namespace kotlin::alloc { + +FixedBlockPage* FixedBlockPage::Create(uint32_t blockSize) noexcept { + CustomAllocInfo("FixedBlockPage::Create(%u)", blockSize); + RuntimeAssert(blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE, "blockSize too large for FixedBlockPage"); + return new (SafeAlloc(FIXED_BLOCK_PAGE_SIZE)) FixedBlockPage(blockSize); +} + +void FixedBlockPage::Destroy() noexcept { + std_support::free(this); +} + +FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) { + CustomAllocInfo("FixedBlockPage(%p)::FixedBlockPage(%u)", this, blockSize); + nextFree_ = cells_; + FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); + for (FixedBlockCell* cell = cells_; cell < end; cell = cell->nextFree) { + cell->nextFree = cell + blockSize; + } +} + +uint8_t* FixedBlockPage::TryAllocate() noexcept { + FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); + FixedBlockCell* freeBlock = nextFree_; + if (freeBlock >= end) { + return nullptr; + } + nextFree_ = freeBlock->nextFree; + CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock); + return freeBlock->data; +} + +bool FixedBlockPage::Sweep() noexcept { + CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this); + // `end` is after the last legal allocation of a block, but does not + // necessarily match an actual block starting point. + FixedBlockCell* end = cells_ + (FIXED_BLOCK_PAGE_CELL_COUNT + 1 - blockSize_); + bool alive = false; + FixedBlockCell** nextFree = &nextFree_; + for (FixedBlockCell* cell = cells_; cell < end; cell += blockSize_) { + // If the current cell is free, move on. + if (cell == *nextFree) { + nextFree = &cell->nextFree; + continue; + } + // If the current cell was marked, it's alive, and the whole page is alive. + if (TryResetMark(cell)) { + alive = true; + continue; + } + CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell); + // Free the current block and insert it into the free list. + cell->nextFree = *nextFree; + *nextFree = cell; + nextFree = &cell->nextFree; + } + return alive; +} + +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp similarity index 54% rename from kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.hpp rename to kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp index c33c6dab300..95a0b532f0d 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp @@ -3,8 +3,8 @@ * that can be found in the LICENSE file. */ -#ifndef CUSTOM_ALLOC_CPP_SMALLPAGE_HPP_ -#define CUSTOM_ALLOC_CPP_SMALLPAGE_HPP_ +#ifndef CUSTOM_ALLOC_CPP_FIXEDBLOCKPAGE_HPP_ +#define CUSTOM_ALLOC_CPP_FIXEDBLOCKPAGE_HPP_ #include #include @@ -13,17 +13,17 @@ namespace kotlin::alloc { -struct alignas(8) SmallCell { - // The SmallCell either contains data or a pointer to the next free cell +struct alignas(8) FixedBlockCell { + // The FixedBlockCell either contains data or a pointer to the next free cell union { uint8_t data[]; - SmallCell* nextFree; + FixedBlockCell* nextFree; }; }; -class alignas(8) SmallPage { +class alignas(8) FixedBlockPage { public: - static SmallPage* Create(uint32_t blockSize) noexcept; + static FixedBlockPage* Create(uint32_t blockSize) noexcept; void Destroy() noexcept; @@ -33,15 +33,15 @@ public: bool Sweep() noexcept; private: - friend class AtomicStack; + friend class AtomicStack; - explicit SmallPage(uint32_t blockSize) noexcept; + explicit FixedBlockPage(uint32_t blockSize) noexcept; // Used for linking pages together in `pages` queue or in `unswept` queue. - SmallPage* next_; + FixedBlockPage* next_; uint32_t blockSize_; - SmallCell* nextFree_; - SmallCell cells_[]; + FixedBlockCell* nextFree_; + FixedBlockCell cells_[]; }; } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp similarity index 54% rename from kotlin-native/runtime/src/custom_alloc/cpp/SmallPageTest.cpp rename to kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp index e2fbaddaf15..554d3ae1431 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp @@ -8,12 +8,12 @@ #include "Cell.hpp" #include "CustomAllocConstants.hpp" #include "gtest/gtest.h" -#include "SmallPage.hpp" +#include "FixedBlockPage.hpp" #include "TypeInfo.h" namespace { -using SmallPage = typename kotlin::alloc::SmallPage; +using FixedBlockPage = typename kotlin::alloc::FixedBlockPage; TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer @@ -21,7 +21,7 @@ void mark(void* obj) { reinterpret_cast(obj)[0] = 1; } -uint8_t* alloc(SmallPage* page, size_t blockSize) { +uint8_t* alloc(FixedBlockPage* page, size_t blockSize) { uint8_t* ptr = page->TryAllocate(); if (ptr) { memset(ptr, 0, 8 * blockSize); @@ -30,9 +30,9 @@ uint8_t* alloc(SmallPage* page, size_t blockSize) { return ptr; } -TEST(CustomAllocTest, SmallPageConsequtiveAlloc) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* prev = alloc(page, size); uint8_t* cur; while ((cur = alloc(page, size))) { @@ -43,28 +43,28 @@ TEST(CustomAllocTest, SmallPageConsequtiveAlloc) { } } -TEST(CustomAllocTest, SmallPageSweepEmptyPage) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); EXPECT_FALSE(page->Sweep()); page->Destroy(); } } -TEST(CustomAllocTest, SmallPageSweepFullUnmarkedPage) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); uint32_t count = 0; while (alloc(page, size)) ++count; - EXPECT_EQ(count, SMALL_PAGE_CELL_COUNT / size); + EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size); EXPECT_FALSE(page->Sweep()); page->Destroy(); } } -TEST(CustomAllocTest, SmallPageSweepSingleMarked) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); mark(ptr); EXPECT_TRUE(page->Sweep()); @@ -72,9 +72,9 @@ TEST(CustomAllocTest, SmallPageSweepSingleMarked) { } } -TEST(CustomAllocTest, SmallPageSweepSingleReuse) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); EXPECT_FALSE(page->Sweep()); EXPECT_EQ(alloc(page, size), ptr); @@ -82,9 +82,9 @@ TEST(CustomAllocTest, SmallPageSweepSingleReuse) { } } -TEST(CustomAllocTest, SmallPageSweepReuse) { - for (uint32_t size = 2; size <= SMALL_PAGE_MAX_BLOCK_SIZE; ++size) { - SmallPage* page = SmallPage::Create(size); +TEST(CustomAllocTest, FixedBlockPageSweepReuse) { + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr; for (int count = 0; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); @@ -94,7 +94,7 @@ TEST(CustomAllocTest, SmallPageSweepReuse) { for (; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); } - EXPECT_EQ(count, SMALL_PAGE_CELL_COUNT / size / 2); + EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size / 2); page->Destroy(); } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index aab0c7646f8..2a9075e0fa7 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -32,21 +32,21 @@ void Heap::PrepareForGC() noexcept { thread.gc().impl().alloc().PrepareForGC(); } - mediumPages_.PrepareForGC(); - largePages_.PrepareForGC(); - for (int blockSize = 0; blockSize <= SMALL_PAGE_MAX_BLOCK_SIZE; ++blockSize) { - smallPages_[blockSize].PrepareForGC(); + nextFitPages_.PrepareForGC(); + singleObjectPages_.PrepareForGC(); + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + fixedBlockPages_[blockSize].PrepareForGC(); } usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_)); } void Heap::Sweep() noexcept { CustomAllocDebug("Heap::Sweep()"); - for (int blockSize = 0; blockSize <= SMALL_PAGE_MAX_BLOCK_SIZE; ++blockSize) { - smallPages_[blockSize].Sweep(); + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + fixedBlockPages_[blockSize].Sweep(); } - mediumPages_.Sweep(); - largePages_.SweepAndFree(); + nextFitPages_.Sweep(); + singleObjectPages_.SweepAndFree(); } AtomicStack Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept { @@ -64,19 +64,19 @@ AtomicStack Heap::SweepExtraObjects(gc::GCHandle gcHandle) noex return finalizerQueue; } -MediumPage* Heap::GetMediumPage(uint32_t cellCount) noexcept { - CustomAllocDebug("Heap::GetMediumPage()"); - return mediumPages_.GetPage(cellCount); +NextFitPage* Heap::GetNextFitPage(uint32_t cellCount) noexcept { + CustomAllocDebug("Heap::GetNextFitPage()"); + return nextFitPages_.GetPage(cellCount); } -SmallPage* Heap::GetSmallPage(uint32_t cellCount) noexcept { - CustomAllocDebug("Heap::GetSmallPage()"); - return smallPages_[cellCount].GetPage(cellCount); +FixedBlockPage* Heap::GetFixedBlockPage(uint32_t cellCount) noexcept { + CustomAllocDebug("Heap::GetFixedBlockPage()"); + return fixedBlockPages_[cellCount].GetPage(cellCount); } -LargePage* Heap::GetLargePage(uint64_t cellCount) noexcept { - CustomAllocInfo("CustomAllocator::AllocateInLargePage(%" PRIu64 ")", cellCount); - return largePages_.NewPage(cellCount); +SingleObjectPage* Heap::GetSingleObjectPage(uint64_t cellCount) noexcept { + CustomAllocInfo("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount); + return singleObjectPages_.NewPage(cellCount); } ExtraObjectPage* Heap::GetExtraObjectPage() noexcept { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp index 0840dcb6a1b..0325a0ad34c 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp @@ -13,10 +13,10 @@ #include "CustomAllocConstants.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" -#include "LargePage.hpp" -#include "MediumPage.hpp" +#include "SingleObjectPage.hpp" +#include "NextFitPage.hpp" #include "PageStore.hpp" -#include "SmallPage.hpp" +#include "FixedBlockPage.hpp" namespace kotlin::alloc { @@ -34,15 +34,15 @@ public: AtomicStack SweepExtraObjects(gc::GCHandle gcHandle) noexcept; - SmallPage* GetSmallPage(uint32_t cellCount) noexcept; - MediumPage* GetMediumPage(uint32_t cellCount) noexcept; - LargePage* GetLargePage(uint64_t cellCount) noexcept; + FixedBlockPage* GetFixedBlockPage(uint32_t cellCount) noexcept; + NextFitPage* GetNextFitPage(uint32_t cellCount) noexcept; + SingleObjectPage* GetSingleObjectPage(uint64_t cellCount) noexcept; ExtraObjectPage* GetExtraObjectPage() noexcept; private: - PageStore smallPages_[SMALL_PAGE_MAX_BLOCK_SIZE + 1]; - PageStore mediumPages_; - PageStore largePages_; + PageStore fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; + PageStore nextFitPages_; + PageStore singleObjectPages_; AtomicStack extraObjectPages_; AtomicStack usedExtraObjectPages_; }; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp index 25f45e4dd8d..9ed2e6e2502 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp @@ -7,17 +7,17 @@ #include #include -#include "LargePage.hpp" +#include "SingleObjectPage.hpp" #include "gtest/gtest.h" #include "Heap.hpp" -#include "SmallPage.hpp" +#include "FixedBlockPage.hpp" namespace { using Heap = typename kotlin::alloc::Heap; -using SmallPage = typename kotlin::alloc::SmallPage; -using MediumPage = typename kotlin::alloc::MediumPage; -using LargePage = typename kotlin::alloc::LargePage; +using FixedBlockPage = typename kotlin::alloc::FixedBlockPage; +using NextFitPage = typename kotlin::alloc::NextFitPage; +using SingleObjectPage = typename kotlin::alloc::SingleObjectPage; inline constexpr int MIN_BLOCK_SIZE = 2; @@ -25,32 +25,32 @@ void mark(void* obj) { reinterpret_cast(obj)[0] = 1; } -TEST(CustomAllocTest, HeapReuseSmallPages) { +TEST(CustomAllocTest, HeapReuseFixedBlockPages) { Heap heap; const int MIN = MIN_BLOCK_SIZE; - const int MAX = SMALL_PAGE_MAX_BLOCK_SIZE + 1; - SmallPage* pages[MAX]; + const int MAX = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1; + FixedBlockPage* pages[MAX]; for (int blocks = MIN; blocks < MAX; ++blocks) { - pages[blocks] = heap.GetSmallPage(blocks); + pages[blocks] = heap.GetFixedBlockPage(blocks); void* obj = pages[blocks]->TryAllocate(); mark(obj); // to make the page survive a sweep } heap.PrepareForGC(); heap.Sweep(); for (int blocks = MIN; blocks < MAX; ++blocks) { - EXPECT_EQ(pages[blocks], heap.GetSmallPage(blocks)); + EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks)); } } -TEST(CustomAllocTest, HeapReuseMediumPages) { +TEST(CustomAllocTest, HeapReuseNextFitPages) { Heap heap; - const uint32_t BLOCKSIZE = SMALL_PAGE_MAX_BLOCK_SIZE + 42; - MediumPage* page = heap.GetMediumPage(BLOCKSIZE); + const uint32_t BLOCKSIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 42; + NextFitPage* page = heap.GetNextFitPage(BLOCKSIZE); void* obj = page->TryAllocate(BLOCKSIZE); mark(obj); // to make the page survive a sweep heap.PrepareForGC(); heap.Sweep(); - EXPECT_EQ(page, heap.GetMediumPage(BLOCKSIZE)); + EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE)); } } // namespace diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/LargePage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/LargePage.cpp deleted file mode 100644 index 0b2943b81ff..00000000000 --- a/kotlin-native/runtime/src/custom_alloc/cpp/LargePage.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "LargePage.hpp" - -#include -#include - -#include "CustomLogging.hpp" -#include "CustomAllocConstants.hpp" -#include "GCApi.hpp" - -namespace kotlin::alloc { - -LargePage* LargePage::Create(uint64_t cellCount) noexcept { - CustomAllocInfo("LargePage::Create(%" PRIu64 ")", cellCount); - RuntimeAssert(cellCount > MEDIUM_PAGE_MAX_BLOCK_SIZE, "blockSize too small for large page"); - uint64_t size = sizeof(LargePage) + cellCount * sizeof(uint64_t); - return new (SafeAlloc(size)) LargePage(); -} - -void LargePage::Destroy() noexcept { - std_support::free(this); -} - -uint8_t* LargePage::Data() noexcept { - return data_; -} - -uint8_t* LargePage::TryAllocate() noexcept { - if (isAllocated_) return nullptr; - isAllocated_ = true; - return Data(); -} - -bool LargePage::Sweep() noexcept { - CustomAllocDebug("LargePage@%p::Sweep()", this); - if (!TryResetMark(Data())) { - isAllocated_ = false; - return false; - } - return true; -} - -} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp similarity index 63% rename from kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.cpp rename to kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index 1b84f31e516..6ddb52384fe 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -3,7 +3,7 @@ * that can be found in the LICENSE file. */ -#include "MediumPage.hpp" +#include "NextFitPage.hpp" #include #include @@ -14,23 +14,23 @@ namespace kotlin::alloc { -MediumPage* MediumPage::Create(uint32_t cellCount) noexcept { - CustomAllocInfo("MediumPage::Create(%u)", cellCount); - RuntimeAssert(cellCount < MEDIUM_PAGE_CELL_COUNT, "cellCount is too large for medium page"); - return new (SafeAlloc(MEDIUM_PAGE_SIZE)) MediumPage(cellCount); +NextFitPage* NextFitPage::Create(uint32_t cellCount) noexcept { + CustomAllocInfo("NextFitPage::Create(%u)", cellCount); + RuntimeAssert(cellCount < NEXT_FIT_PAGE_CELL_COUNT, "cellCount is too large for NextFitPage"); + return new (SafeAlloc(NEXT_FIT_PAGE_SIZE)) NextFitPage(cellCount); } -void MediumPage::Destroy() noexcept { +void NextFitPage::Destroy() noexcept { std_support::free(this); } -MediumPage::MediumPage(uint32_t cellCount) noexcept : curBlock_(cells_) { +NextFitPage::NextFitPage(uint32_t cellCount) noexcept : curBlock_(cells_) { cells_[0] = Cell(0); // Size 0 ensures any actual use would break - cells_[1] = Cell(MEDIUM_PAGE_CELL_COUNT - 1); + cells_[1] = Cell(NEXT_FIT_PAGE_CELL_COUNT - 1); } -uint8_t* MediumPage::TryAllocate(uint32_t blockSize) noexcept { - CustomAllocDebug("MediumPage@%p::TryAllocate(%u)", this, blockSize); +uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept { + CustomAllocDebug("NextFitPage@%p::TryAllocate(%u)", this, blockSize); // +1 accounts for header, since cell->size also includes header cell uint32_t cellsNeeded = blockSize + 1; uint8_t* block = curBlock_->TryAllocate(cellsNeeded); @@ -39,9 +39,9 @@ uint8_t* MediumPage::TryAllocate(uint32_t blockSize) noexcept { return curBlock_->TryAllocate(cellsNeeded); } -bool MediumPage::Sweep() noexcept { - CustomAllocDebug("MediumPage@%p::Sweep()", this); - Cell* end = cells_ + MEDIUM_PAGE_CELL_COUNT; +bool NextFitPage::Sweep() noexcept { + CustomAllocDebug("NextFitPage@%p::Sweep()", this); + Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT; bool alive = false; for (Cell* block = cells_ + 1; block != end; block = block->Next()) { if (block->isAllocated_) { @@ -64,10 +64,10 @@ bool MediumPage::Sweep() noexcept { return alive; } -void MediumPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept { - CustomAllocDebug("MediumPage@%p::UpdateCurBlock(%u)", this, cellsNeeded); +void NextFitPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept { + CustomAllocDebug("NextFitPage@%p::UpdateCurBlock(%u)", this, cellsNeeded); if (curBlock_ == cells_) curBlock_ = cells_ + 1; // only used as a starting point - Cell* end = cells_ + MEDIUM_PAGE_CELL_COUNT; + Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT; Cell* maxBlock = cells_; // size 0 block for (Cell* block = curBlock_; block != end; block = block->Next()) { if (!block->isAllocated_ && block->size_ > maxBlock->size_) { @@ -78,7 +78,7 @@ void MediumPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept { } } } - CustomAllocDebug("MediumPage@%p::UpdateCurBlock: starting from beginning", this); + CustomAllocDebug("NextFitPage@%p::UpdateCurBlock: starting from beginning", this); for (Cell* block = cells_ + 1; block != curBlock_; block = block->Next()) { if (!block->isAllocated_ && block->size_ > maxBlock->size_) { maxBlock = block; @@ -91,12 +91,12 @@ void MediumPage::UpdateCurBlock(uint32_t cellsNeeded) noexcept { curBlock_ = maxBlock; } -bool MediumPage::CheckInvariants() noexcept { - if (curBlock_ < cells_ || curBlock_ >= cells_ + MEDIUM_PAGE_CELL_COUNT) return false; +bool NextFitPage::CheckInvariants() noexcept { + if (curBlock_ < cells_ || curBlock_ >= cells_ + NEXT_FIT_PAGE_CELL_COUNT) return false; for (Cell* cur = cells_ + 1;; cur = cur->Next()) { if (cur->Next() <= cur) return false; - if (cur->Next() > cells_ + MEDIUM_PAGE_CELL_COUNT) return false; - if (cur->Next() == cells_ + MEDIUM_PAGE_CELL_COUNT) return true; + if (cur->Next() > cells_ + NEXT_FIT_PAGE_CELL_COUNT) return false; + if (cur->Next() == cells_ + NEXT_FIT_PAGE_CELL_COUNT) return true; } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp similarity index 74% rename from kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.hpp rename to kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp index 3908ad25746..b7a9563e60c 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp @@ -3,8 +3,8 @@ * that can be found in the LICENSE file. */ -#ifndef CUSTOM_ALLOC_CPP_MEDIUMPAGE_HPP_ -#define CUSTOM_ALLOC_CPP_MEDIUMPAGE_HPP_ +#ifndef CUSTOM_ALLOC_CPP_NEXTFITPAGE_HPP_ +#define CUSTOM_ALLOC_CPP_NEXTFITPAGE_HPP_ #include #include @@ -14,9 +14,9 @@ namespace kotlin::alloc { -class alignas(8) MediumPage { +class alignas(8) NextFitPage { public: - static MediumPage* Create(uint32_t cellCount) noexcept; + static NextFitPage* Create(uint32_t cellCount) noexcept; void Destroy() noexcept; @@ -29,14 +29,14 @@ public: bool CheckInvariants() noexcept; private: - MediumPage(uint32_t cellCount) noexcept; + NextFitPage(uint32_t cellCount) noexcept; // Looks for a block big enough to hold cellsNeeded. If none big enough is // found, update to the largest one. void UpdateCurBlock(uint32_t cellsNeeded) noexcept; - friend class AtomicStack; - MediumPage* next_; + friend class AtomicStack; + NextFitPage* next_; Cell* curBlock_; Cell cells_[]; // cells_[0] is reserved for an empty block diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp similarity index 68% rename from kotlin-native/runtime/src/custom_alloc/cpp/MediumPageTest.cpp rename to kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp index 886be8f1c96..1bb8306b770 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/MediumPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp @@ -9,23 +9,23 @@ #include "Cell.hpp" #include "CustomAllocConstants.hpp" #include "gtest/gtest.h" -#include "MediumPage.hpp" +#include "NextFitPage.hpp" #include "TypeInfo.h" namespace { -using MediumPage = typename kotlin::alloc::MediumPage; +using NextFitPage = typename kotlin::alloc::NextFitPage; using Cell = typename kotlin::alloc::Cell; TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer -inline constexpr const size_t MIN_BLOCK_SIZE = SMALL_PAGE_MAX_BLOCK_SIZE + 1; +inline constexpr const size_t MIN_BLOCK_SIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1; void mark(void* obj) { reinterpret_cast(obj)[0] = 1; } -uint8_t* alloc(MediumPage* page, uint32_t blockSize) { +uint8_t* alloc(NextFitPage* page, uint32_t blockSize) { uint8_t* ptr = page->TryAllocate(blockSize); if (!page->CheckInvariants()) { ADD_FAILURE(); @@ -41,8 +41,8 @@ uint8_t* alloc(MediumPage* page, uint32_t blockSize) { return ptr; } -TEST(CustomAllocTest, MediumPageAlloc) { - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); +TEST(CustomAllocTest, NextFitPageAlloc) { + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); uint8_t* p1 = alloc(page, MIN_BLOCK_SIZE); uint8_t* p2 = alloc(page, MIN_BLOCK_SIZE); uint64_t dist = abs(p1 - p2); @@ -50,33 +50,33 @@ TEST(CustomAllocTest, MediumPageAlloc) { page->Destroy(); } -TEST(CustomAllocTest, MediumPageSweepEmptyPage) { - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); +TEST(CustomAllocTest, NextFitPageSweepEmptyPage) { + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); EXPECT_FALSE(page->Sweep()); page->Destroy(); } -TEST(CustomAllocTest, MediumPageSweepFullUnmarkedPage) { +TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) { for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {} EXPECT_FALSE(page->Sweep()); page->Destroy(); } } -TEST(CustomAllocTest, MediumPageSweepSingleMarked) { - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); +TEST(CustomAllocTest, NextFitPageSweepSingleMarked) { + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); mark(alloc(page, MIN_BLOCK_SIZE)); EXPECT_TRUE(page->Sweep()); page->Destroy(); } -TEST(CustomAllocTest, MediumPageSweepSingleReuse) { +TEST(CustomAllocTest, NextFitPageSweepSingleReuse) { for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); int count1 = 0; while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1; EXPECT_FALSE(page->Sweep()); @@ -88,10 +88,10 @@ TEST(CustomAllocTest, MediumPageSweepSingleReuse) { } } -TEST(CustomAllocTest, MediumPageSweepReuse) { +TEST(CustomAllocTest, NextFitPageSweepReuse) { for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); int unmarked = 0; while (true) { uint8_t* ptr = alloc(page, MIN_BLOCK_SIZE); @@ -110,11 +110,11 @@ TEST(CustomAllocTest, MediumPageSweepReuse) { } } -TEST(CustomAllocTest, MediumPageSweepCoallesce) { - MediumPage* page = MediumPage::Create(MIN_BLOCK_SIZE); - EXPECT_TRUE(alloc(page, (MEDIUM_PAGE_CELL_COUNT-1) / 2 - 1)); +TEST(CustomAllocTest, NextFitPageSweepCoallesce) { + NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); + EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1)); EXPECT_FALSE(page->Sweep()); - EXPECT_TRUE(alloc(page, (MEDIUM_PAGE_CELL_COUNT-1) - 1)); + EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp new file mode 100644 index 00000000000..896c6922243 --- /dev/null +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "SingleObjectPage.hpp" + +#include +#include + +#include "CustomLogging.hpp" +#include "CustomAllocConstants.hpp" +#include "GCApi.hpp" + +namespace kotlin::alloc { + +SingleObjectPage* SingleObjectPage::Create(uint64_t cellCount) noexcept { + CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount); + RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage"); + uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t); + return new (SafeAlloc(size)) SingleObjectPage(); +} + +void SingleObjectPage::Destroy() noexcept { + std_support::free(this); +} + +uint8_t* SingleObjectPage::Data() noexcept { + return data_; +} + +uint8_t* SingleObjectPage::TryAllocate() noexcept { + if (isAllocated_) return nullptr; + isAllocated_ = true; + return Data(); +} + +bool SingleObjectPage::Sweep() noexcept { + CustomAllocDebug("SingleObjectPage@%p::Sweep()", this); + if (!TryResetMark(Data())) { + isAllocated_ = false; + return false; + } + return true; +} + +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/LargePage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp similarity index 65% rename from kotlin-native/runtime/src/custom_alloc/cpp/LargePage.hpp rename to kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index aad7992efb6..c42500bf233 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/LargePage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -3,20 +3,19 @@ * that can be found in the LICENSE file. */ -#ifndef CUSTOM_ALLOC_CPP_LARGEPAGE_HPP_ -#define CUSTOM_ALLOC_CPP_LARGEPAGE_HPP_ +#ifndef CUSTOM_ALLOC_CPP_SINGLEOBJECTPAGE_HPP_ +#define CUSTOM_ALLOC_CPP_SINGLEOBJECTPAGE_HPP_ #include #include #include "AtomicStack.hpp" -#include "MediumPage.hpp" namespace kotlin::alloc { -class alignas(8) LargePage { +class alignas(8) SingleObjectPage { public: - static LargePage* Create(uint64_t cellCount) noexcept; + static SingleObjectPage* Create(uint64_t cellCount) noexcept; void Destroy() noexcept; @@ -27,8 +26,8 @@ public: bool Sweep() noexcept; private: - friend class AtomicStack; - LargePage* next_; + friend class AtomicStack; + SingleObjectPage* next_; bool isAllocated_ = false; struct alignas(8) { uint8_t data_[]; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/LargePageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp similarity index 64% rename from kotlin-native/runtime/src/custom_alloc/cpp/LargePageTest.cpp rename to kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp index acde849d9ae..fb6f3764d93 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/LargePageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp @@ -8,38 +8,38 @@ #include "CustomAllocConstants.hpp" #include "gtest/gtest.h" -#include "LargePage.hpp" +#include "SingleObjectPage.hpp" #include "TypeInfo.h" namespace { -using LargePage = typename kotlin::alloc::LargePage; +using SingleObjectPage = typename kotlin::alloc::SingleObjectPage; TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer -#define MIN_BLOCK_SIZE MEDIUM_PAGE_CELL_COUNT +#define MIN_BLOCK_SIZE NEXT_FIT_PAGE_CELL_COUNT void mark(void* obj) { reinterpret_cast(obj)[0] = 1; } -LargePage* alloc(uint64_t blockSize) { - LargePage* page = LargePage::Create(blockSize); +SingleObjectPage* alloc(uint64_t blockSize) { + SingleObjectPage* page = SingleObjectPage::Create(blockSize); uint64_t* ptr = reinterpret_cast(page->TryAllocate()); memset(ptr, 0, 8 * blockSize); ptr[1] = reinterpret_cast(&fakeType); return page; } -TEST(CustomAllocTest, LargePageSweepEmptyPage) { - LargePage* page = alloc(MIN_BLOCK_SIZE); +TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) { + SingleObjectPage* page = alloc(MIN_BLOCK_SIZE); EXPECT_TRUE(page); EXPECT_FALSE(page->Sweep()); page->Destroy(); } -TEST(CustomAllocTest, LargePageSweepFullPage) { - LargePage* page = alloc(MIN_BLOCK_SIZE); +TEST(CustomAllocTest, SingleObjectPageSweepFullPage) { + SingleObjectPage* page = alloc(MIN_BLOCK_SIZE); EXPECT_TRUE(page); EXPECT_TRUE(page->Data()); mark(page->Data()); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.cpp deleted file mode 100644 index f35e01c56e7..00000000000 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SmallPage.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "SmallPage.hpp" - -#include -#include -#include -#include - -#include "CustomLogging.hpp" -#include "CustomAllocConstants.hpp" -#include "GCApi.hpp" - -namespace kotlin::alloc { - -SmallPage* SmallPage::Create(uint32_t blockSize) noexcept { - CustomAllocInfo("SmallPage::Create(%u)", blockSize); - RuntimeAssert(blockSize <= SMALL_PAGE_MAX_BLOCK_SIZE, "blockSize too large for small page"); - return new (SafeAlloc(SMALL_PAGE_SIZE)) SmallPage(blockSize); -} - -void SmallPage::Destroy() noexcept { - std_support::free(this); -} - -SmallPage::SmallPage(uint32_t blockSize) noexcept : blockSize_(blockSize) { - CustomAllocInfo("SmallPage(%p)::SmallPage(%u)", this, blockSize); - nextFree_ = cells_; - SmallCell* end = cells_ + (SMALL_PAGE_CELL_COUNT + 1 - blockSize_); - for (SmallCell* cell = cells_; cell < end; cell = cell->nextFree) { - cell->nextFree = cell + blockSize; - } -} - -uint8_t* SmallPage::TryAllocate() noexcept { - SmallCell* end = cells_ + (SMALL_PAGE_CELL_COUNT + 1 - blockSize_); - SmallCell* freeBlock = nextFree_; - if (freeBlock >= end) { - return nullptr; - } - nextFree_ = freeBlock->nextFree; - CustomAllocDebug("SmallPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock); - return freeBlock->data; -} - -bool SmallPage::Sweep() noexcept { - CustomAllocInfo("SmallPage(%p)::Sweep()", this); - // `end` is after the last legal allocation of a block, but does not - // necessarily match an actual block starting point. - SmallCell* end = cells_ + (SMALL_PAGE_CELL_COUNT + 1 - blockSize_); - bool alive = false; - SmallCell** nextFree = &nextFree_; - for (SmallCell* cell = cells_; cell < end; cell += blockSize_) { - // If the current cell is free, move on. - if (cell == *nextFree) { - nextFree = &cell->nextFree; - continue; - } - // If the current cell was marked, it's alive, and the whole page is alive. - if (TryResetMark(cell)) { - alive = true; - continue; - } - CustomAllocInfo("SmallPage(%p)::Sweep: reclaim %p", this, cell); - // Free the current block and insert it into the free list. - cell->nextFree = *nextFree; - *nextFree = cell; - nextFree = &cell->nextFree; - } - return alive; -} - -} // namespace kotlin::alloc