diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 0d376e8e9ae..7f538f7677c 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -298,6 +298,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val runtimeNativeLibraries: List = mutableListOf().apply { if (debug) add("debug.bc") + add("mm.bc") + add("common_alloc.bc") add("common_gc.bc") add("common_gcScheduler.bc") when (gcSchedulerType) { @@ -315,14 +317,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } } if (allocationMode == AllocationMode.CUSTOM) { - add("experimental_memory_manager_custom.bc") when (gc) { GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc") GC.NOOP -> add("noop_gc_custom.bc") GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc") } } else { - add("experimental_memory_manager.bc") when (gc) { GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc") GC.NOOP -> add("noop_gc.bc") @@ -339,10 +339,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } when (allocationMode) { AllocationMode.MIMALLOC -> { - add("opt_alloc.bc") + add("legacy_alloc.bc") + add("mimalloc_alloc.bc") add("mimalloc.bc") } AllocationMode.STD -> { + add("legacy_alloc.bc") add("std_alloc.bc") } AllocationMode.CUSTOM -> { diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index ae2090d47dc..0c6c5289857 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -150,15 +150,25 @@ bitcode { } } + module("common_alloc") { + srcRoot.set(layout.projectDirectory.dir("src/alloc/common")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + } + } + module("std_alloc") { - headersDirs.from(files("src/main/cpp")) + srcRoot.set(layout.projectDirectory.dir("src/alloc/std")) + headersDirs.from(files("src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} } } module("custom_alloc") { - headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp")) + srcRoot.set(layout.projectDirectory.dir("src/alloc/custom")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -167,8 +177,9 @@ bitcode { compilerArgs.add("-DCUSTOM_ALLOCATOR") } - module("opt_alloc") { - headersDirs.from(files("src/main/cpp")) + module("mimalloc_alloc") { + srcRoot.set(layout.projectDirectory.dir("src/alloc/mimalloc")) + headersDirs.from(files("src/mimalloc/c/include", "src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} } @@ -176,6 +187,15 @@ bitcode { compilerArgs.add("-DKONAN_MI_MALLOC=1") } + module("legacy_alloc") { + srcRoot.set(layout.projectDirectory.dir("src/alloc/legacy")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + sourceSets { + main {} + test {} + } + } + module("exceptionsSupport") { srcRoot.set(layout.projectDirectory.dir("src/exceptions_support")) headersDirs.from(files("src/main/cpp")) @@ -227,9 +247,8 @@ bitcode { } } - module("experimental_memory_manager") { - srcRoot.set(layout.projectDirectory.dir("src/mm")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp")) + module("mm") { + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp")) sourceSets { main {} testFixtures {} @@ -237,21 +256,9 @@ bitcode { } } - module("experimental_memory_manager_custom") { - srcRoot.set(layout.projectDirectory.dir("src/mm")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp", "src/custom_alloc/cpp")) - sourceSets { - main {} - testFixtures {} - test {} - } - - compilerArgs.add("-DCUSTOM_ALLOCATOR") - } - module("common_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/common")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -260,7 +267,7 @@ bitcode { module("noop_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/noop")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} testFixtures {} @@ -269,7 +276,7 @@ bitcode { module("noop_gc_custom") { srcRoot.set(layout.projectDirectory.dir("src/gc/noop")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} testFixtures {} @@ -280,7 +287,7 @@ bitcode { module("same_thread_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/stms")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} testFixtures {} @@ -290,7 +297,7 @@ bitcode { module("same_thread_ms_gc_custom") { srcRoot.set(layout.projectDirectory.dir("src/gc/stms")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} testFixtures {} @@ -302,7 +309,7 @@ bitcode { module("concurrent_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/cms")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} testFixtures {} @@ -312,7 +319,7 @@ bitcode { module("concurrent_ms_gc_custom") { srcRoot.set(layout.projectDirectory.dir("src/gc/cms")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} testFixtures {} @@ -324,7 +331,7 @@ bitcode { module("common_gcScheduler") { srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/common")) - headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -333,7 +340,7 @@ bitcode { module("manual_gcScheduler") { srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/manual")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} } @@ -341,7 +348,7 @@ bitcode { module("adaptive_gcScheduler") { srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/adaptive")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -350,7 +357,7 @@ bitcode { module("aggressive_gcScheduler") { srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/aggressive")) - headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) + headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} test {} @@ -359,56 +366,56 @@ bitcode { testsGroup("custom_alloc_runtime_tests") { testedModules.addAll("custom_alloc") - testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "objc") + testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc_custom", "objc") } testsGroup("experimentalMM_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_custom_alloc_runtime_tests") { - testedModules.addAll("experimental_memory_manager_custom", "same_thread_ms_gc_custom") - testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + testedModules.addAll("mm", "same_thread_ms_gc_custom") + testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") } testsGroup("experimentalMM_cms_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_cms_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") { - testedModules.addAll("experimental_memory_manager_custom", "concurrent_ms_gc_custom") - testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + testedModules.addAll("mm", "concurrent_ms_gc_custom") + testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") } testsGroup("experimentalMM_noop_mimalloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_noop_std_alloc_runtime_tests") { - testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc") + testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc") } testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") { - testedModules.addAll("experimental_memory_manager_custom", "noop_gc_custom") - testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + testedModules.addAll("mm", "noop_gc_custom") + testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") } testsGroup("aggressive_gcScheduler_runtime_tests") { testedModules.addAll("aggressive_gcScheduler") - testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc") + testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc") } testsGroup("adaptive_gcScheduler_runtime_tests") { testedModules.addAll("adaptive_gcScheduler") - testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc") + testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc") } } } diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.cpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.cpp new file mode 100644 index 00000000000..5a3fd42d805 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.cpp @@ -0,0 +1,6 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Allocator.hpp" diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp new file mode 100644 index 00000000000..1cc7c58c5c8 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp @@ -0,0 +1,16 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +namespace kotlin::alloc { + +// TODO: Build `Allocator`, `Allocator::ThreadData` like with `gc`, `gcScheduler`, +// and move allocator-specific data there. + +void initObjectPool() noexcept; +void compactObjectPoolInCurrentThread() noexcept; + +} diff --git a/kotlin-native/runtime/src/alloc/custom/README.md b/kotlin-native/runtime/src/alloc/custom/README.md new file mode 100644 index 00000000000..6d20c03cc49 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/custom/README.md @@ -0,0 +1,372 @@ +# Overview of the allocator + +This document describes the internals of the custom allocator. The presentation +here is not fully true to the implementation, as the design is still work in +progress. + +The main idea of the custom allocator is to divide system memory into chunks +(pages) that each can be swept independently, in memory consecutive order. +Every allocation ends up as a block of memory inside a page. Each page keeps +track of the size of each block it contains; how this is done depends on the +page type, with different page types optimized for different allocation sizes. +All the memory blocks are consecutive within a page, so the size of a block +also tells where the next block begins. Paired with an additional mechanism +(page type dependent) for determining whether a block is allocated, we can +iterate through the allocated blocks. + +When a thread allocates memory for an object, it will find a page suitable for +the given allocation size. Each thread holds on to a number of pages for the +different size categories. The typical case is that the thread’s current page +for the given size can fit the requested allocation. If that is not the case, a +different page for that size category is requested from the shared allocation +space. The requested page can either be readily available (already prepared by +the GC thread), or it might need to be swept first, or it might be newly +created. + +The GC thread has a new responsibility when using this allocator: While the +mutator threads are paused at the start of GC, the GC thread must prepare the +allocator for sweeping. This does two things. First, it marks all pages as +“needs to be swept before next use”. Second, it releases pages that threads are +holding on to, by clearing the thread local variables for each thread. + +It is possible to have several independent allocation spaces at the same time. +This is currently useful for testing, but is potentially useful in other +settings. + +# Detailed Design + +All allocations are made through a `CustomAllocator` object. + +## [CustomAllocator](cpp/CustomAllocator.hpp) + +```cpp +class CustomAllocator { +public: + CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler); + ObjectHeader* CreateObject(TypeInfo* type); + ArrayHeader* CreateArray(TypeInfo* type, uint32_t count); + ExtraObjectData* CreateExtraObject(); + void PrepareForGc(); + +private: + uint8_t* Allocate(uint64_t cellCount); + uint8_t* AllocateInSingleObjectPage(uint64_t cellCount); + uint8_t* AllocateInNextFitPage(uint32_t cellCount); + uint8_t* AllocateInFixedBlockPage(uint32_t cellCount); + + Heap& heap_; + GCSchedulerThreadData& gcScheduler_; + NextFitPage* nextFitPage_; + FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE]; + ExtraObjectPage* extraObjectPage_; +}; +``` + +The primary responsibility of this class is to delegate each requested +allocation to pages of the appropriate type, based on allocation size. To do +this, it requests pages from the shared allocation space (`Heap`) and stores +pages for later allocations. Each thread thus owns a number of pages for +different allocation sizes, but at most one for each size class. When +allocating, the `CustomAllocator` will first try to allocate in one of its +owned pages. If this fails, it will request a new page for that size class from +a shared `Heap` object. `SingleObjectPages` are never kept by the +`CustomAllocator`, since they are created specifically for a single allocation, +with no extra space. + +## [Heap](cpp/Heap.hpp) + +```cpp +class Heap { +public: + void PrepareForGC(); + + void Sweep(); + + AtomicStack SweepExtraObjects(GCHandle gcHandle); + + FixedBlockPage* GetFixedBlockPage(uint32_t cellCount); + NextFitPage* GetNextFitPage(uint32_t cellCount); + SingleObjectPage* GetSingleObjectPage(uint64_t cellCount); + ExtraObjectPage* GetExtraObjectPage(); + +private: + PageStore fixedBlockPages_[MAX_BLOCK_SIZE]; + PageStore nextFitPages_; + PageStore singleObjectPages_; + AtomicStack extraObjectPages_; + AtomicStack usedExtraObjectPages_; +}; +``` + +A `Heap` object represents a shared allocation space for multiple +`CustomAllocator`s, which can request pages through one of the +`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also +provides a method for sweeping through all blocks that have been allocated in +this heap. The `Heap` object is the synchronization point, and guarantees that +every page is returned at most once. Page ownership is thus implicitly given to +the thread that called the method. The `Heap` object keeps track of all pages, +so there is no need to explicitly return ownership of a page. Internally, a +`Heap` keeps the pages for each size class in a `PageStore`. This means one for +`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes +for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two +`AtomicStack`s, since they require different handling during sweeping. + +## [PageStore](cpp/PageStore.hpp) + +```cpp +template +class PageStore { +public: + void PrepareForGC(); + void Sweep(); + void SweepAndFree(); + PageType* GetPage(uint32_t cellCount); + PageType* NewPage(uint64_t cellCount); + +private: + AtomicStack empty_; + AtomicStack ready_; + AtomicStack used_; + AtomicStack unswept_; +}; +``` + +A PageStore is responsible for keeping track of all pages of a given type and +size class. Each of the pages are in one of four stacks. The stack, that a +given page is in, determines its current state: + +* `unswept_`: have not yet been swept since the last GC cycle. +* `ready_`: are ready for allocation; has been swept by the GC thread. +* `used_`: has been given to some thread for allocation; it might still be used + for allocation, or it might have been discarded with not enough space left. + Will not be used until the next GC cycle. +* `empty_`: same as `ready_`, but does not contain any objects. Will be freed + before the next GC, if not needed before then. + +When a page is requested, the page is taken from `ready_`, if there are any. +Otherwise, an `unswept_` page is taken and swept before returning. If there are +no unswept pages either, an empty page is taken, if there are any. Otherwise a +new page is created in the size category. All returned pages are moved to +`used_`. During the marking phase, all remaining pages in `empty_` are freed, +and all other pages are moved to `unswept_`. The GC thread will go through all +`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one +of the other threads sweeps a page from `unswept_`, it is moved directly to +`used_`, as it is claimed by the `CustomAllocator` that swept it. + +`SingleObjectPage`s are treated slightly differently, because they are created +for one specific single allocation, and not reused when that allocation is +freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without +checking any of the stacks, and during sweeping, they are freed directly rather +than being put into the `empty_` stack. Ideally, there would only be two stacks +in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little +is lost by just using the existing `PageStore` logic used for the other pages. + +## [AtomicStack](cpp/AtomicStack.hpp) + +```cpp +template +class AtomicStack { +public: + PageType* Pop(); + void Push(PageType* elm); + void TransferAllFrom(AtomicStack& src); + bool isEmpty(); + + +private: + std::atomic stack_; +}; +``` + +The only place where atomics are used are in the stacks inside the `PageStore`. +All page classes have a non-atomic next pointer, to be used for linking up in +exactly one stack. `Pop` and `Push` are implemented with compare-and-swap +operations. The class is thread safe, except for if an element is freed while +another thread tries to Pop it from a stack. + +# Page types + +This section is likely to change, given the likely introduction of additional +page types. It also describes some details about which page type is chosen for +a given allocation, which is also likely to change. + +There are four different page types, but they all share the feature that they +can be swept independently. The Sweep methods return whether there were any +live objects in the page after sweeping. If not, the page will be given back to +the OS. + +## [FixedBlockPage](cpp/FixedBlockPage.hpp) + +```cpp +class FixedBlockPage { +public: + FixedBlockPage(uint32_t blockSize) noexcept; + + uint8_t* TryAllocate() noexcept; + + bool Sweep() noexcept; + +private: + FixedBlockPage* next_; + FixedCellRange nextFree_; + uint32_t blockSize_; + uint32_t end_; + FixedBlockCell cells_[]; +}; +}; +``` + +All sufficiently small allocations (currently arbitrary <1KiB) are directed to +a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations +are expected to be in this page type. A `FixedBlockPage` consists of a number +of equally sized blocks, where each allocation will take up exactly one such +block. + +```cpp +struct FixedBlockCell { + union { + uint8_t data[]; + FixedCellRange nextFree; + } +}; + +struct alignas(8) FixedCellRange { + uint32_t first; + uint32_t last; +}; +``` + +Consecutive unallocated cells are represented by a `FixedCellRange`, with +`.first` and `.last` being the inclusive end points of the range of unallocated +cells. The `FixedBlockCell` at the the `.last` index contains a +`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange` +of unallocated ranges thus form a linked list. + +The important point is that all links in this list point forward in the page, so +all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a +`FixedBlockPage` consists of walking the free-list forward, and sweeping all +blocks in between the links, maintaining the free list when blocks are freed. + +Each small page takes up the same amount of space, independent of block size, +so larger block size implies fewer blocks per page. This size is arbitrarily +chosen to be 256 KiB, but this might change. + +## [NextFitPage](cpp/NextFitPage.hpp) + +```cpp +class NextFitPage { +public: + NextFitPage(uint32_t cellCount); + Cell* TryAllocate(uint32_t cellCount); + bool Sweep(); + +private: + NextFitPage* next_; // used by AtomicStack + Cell* curBlock_; + Cell cells_[]; +}; +``` + +Allocations that could theoretically fit in a `FixedBlockPage`, but would +require too large a block size (arbitrary >=1KiB), are allocated in a +`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64 +KiB for experiments). All blocks in a `NextFitPage` have a header that tells how +big the block is, and whether it is allocated or not. There are no gaps between +blocks, so the size of a block also tells where the next block is. The header +information fits inside a 8 byte `Cell`. + +```cpp +class Cell { +public: + Cell(uint32_t size); + uint8_t* TryAllocate(uint32_t cellCount); + +private: + uint32_t isAllocated_; + uint32_t size_; + uint8_t data_[]; +}; +``` + +The page keeps a reference to a currently active block, and will try to bump +allocate inside that block. If allocation does not fit, we move to the next +block that fits. If no block in the page fits the requested size, the page is +abandoned until the next GC. + +## [SingleObjectPage](cpp/SingleObjectPage.hpp) + +```cpp +class SingleObjectPage { +public: + SingleObjectPage(uint64_t cellCount); + bool Sweep(); + +private: + SingleObjectPage* next_; // used by AtomicStack +}; +``` + +Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`, +which only contains that single block of the requested size. They are also +handled slightly differently by both `Heap` and `CustomAllocator`. First off, +`Heap::GetSingleObjectPage` will never check existing pages, and instead just +allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to +any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC +thread. + +## [ExtraObjectPage](cpp/ExtraObjectPage.hpp) + +```cpp +class ExtraObjectPage { +public: + ExtraObjectPage(); + ExtraObjectData* TryAllocate(); + bool Sweep(FinalizerQueue& queue); + + +private: + ExtraObjectPage* next_; // used by AtomicStack + ExtraObjectCell* nextFree_; + ExtraObjectCell cells_[]; +}; +``` + +Extra objects are used for attaching additional data to some objects. This is +used for objects that require special handling during garbage collection: + +* objects with finalizers +* weak references +* interop references + +Extra objects are allocated in `ExtraObjectPage`s, which are very similar to +`FixedBlockPage`s. They primarily differ in how they are swept, since it is +during sweeping of `ExtraObject`s that scheduling of finalization happens. If +an object that requires finalization is found, it is added to the +`FinalizerQueue` given as argument. The cells are also slightly different, in +that they add a new field that allows the cells to be added to the finalizer +queue. + +```cpp +struct ExtraObjectCell { + ExtraObjectCell* next_; // used by AtomicStack + ExtraObjectData data_; +}; +``` + +# Finalizers + +Section like to change. + +In the existing memory model, finalization tasks are found and scheduled during +sweeping of regular objects. The objects to be finalized are chained together +using a pointer in the Node header, added to all allocated objects. This header +is not needed in the custom allocator, apart from linking in the finalization +queue. + +We therefore reintroduce this pointer in a header for `ExtraObjectData`. For +this, we reuse the `ExtraObjectCell` as header for both free list pointer and as +linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`. + +# Enabling the allocator + +The custom allocator is enabled with the compiler flag -Xallocator=custom. diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp new file mode 100644 index 00000000000..1bc03350b2b --- /dev/null +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Allocator.hpp" + +using namespace kotlin; + +void alloc::initObjectPool() noexcept {} + +void alloc::compactObjectPoolInCurrentThread() noexcept {} diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/AtomicStack.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/AtomicStack.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStackTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AtomicStackTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/AtomicStackTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/AtomicStackTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/Cell.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/Cell.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/Cell.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/Cell.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/Cell.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocConstants.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocator.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocatorTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocatorTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomFinalizerProcessor.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomFinalizerProcessor.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomLogging.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/CustomLogging.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPageTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPageTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPageTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/HeapTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPageTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/NextFitPageTest.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.cpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp rename to kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPageTest.cpp similarity index 100% rename from kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp rename to kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPageTest.cpp diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.cpp similarity index 55% rename from kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.cpp index 884b7ac7040..c883a62a962 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.cpp @@ -10,14 +10,14 @@ using namespace kotlin; -mm::ExtraObjectData& mm::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject( +mm::ExtraObjectData& alloc::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject( ObjHeader* baseObject, const TypeInfo* info) noexcept { return **Emplace(baseObject, info); } -void mm::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(ExtraObjectData& data) noexcept { +void alloc::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept { Erase(&Queue::Node::fromValue(data)); } -mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default; -mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default; +alloc::ExtraObjectDataFactory::ExtraObjectDataFactory() = default; +alloc::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default; diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.hpp similarity index 80% rename from kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.hpp index dc1a5e41118..ffaa9edf00a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ExtraObjectDataFactory.hpp @@ -3,8 +3,7 @@ * that can be found in the LICENSE file. */ -#ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H -#define RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H +#pragma once #include "ExtraObjectData.hpp" #include "Memory.h" @@ -12,8 +11,7 @@ #include "ObjectAlloc.hpp" #include "ThreadRegistry.hpp" -namespace kotlin { -namespace mm { +namespace kotlin::alloc { // Registry for extra data, attached to some kotlin objects: weak refs, associated objects, ... class ExtraObjectDataFactory : Pinned { @@ -24,9 +22,9 @@ public: public: explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {} - ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept; + mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept; - void DestroyExtraObjectData(ExtraObjectData& data) noexcept; + void DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept; // Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread // when it's asked by GC to stop. @@ -56,7 +54,4 @@ private: Queue extraObjects_; }; -} // namespace mm -} // namespace kotlin - -#endif // RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp similarity index 98% rename from kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp index a196b968f63..4c813319b58 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp @@ -8,7 +8,7 @@ #include #include -namespace kotlin { +namespace kotlin::alloc { void initObjectPool() noexcept; void* allocateInObjectPool(size_t size) noexcept; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactory.hpp similarity index 99% rename from kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactory.hpp index abbe60ef68d..3c4633ca090 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactory.hpp @@ -3,8 +3,7 @@ * that can be found in the LICENSE file. */ -#ifndef RUNTIME_MM_OBJECT_FACTORY_H -#define RUNTIME_MM_OBJECT_FACTORY_H +#pragma once #include #include @@ -21,8 +20,7 @@ #include "Types.h" #include "Utils.hpp" -namespace kotlin { -namespace mm { +namespace kotlin::alloc { namespace internal { @@ -736,7 +734,4 @@ private: Storage storage_; }; -} // namespace mm -} // namespace kotlin - -#endif // RUNTIME_MM_OBJECT_FACTORY_H +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocator.hpp similarity index 93% rename from kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocator.hpp index 6cc493ccbdc..46ea2a21dd5 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocator.hpp @@ -9,14 +9,13 @@ #include "ObjectAlloc.hpp" -namespace kotlin { -namespace gc { +namespace kotlin::alloc { // TODO: Try to move from custom allocator interface to standard one. // Currently Free method is in the way: it is static to avoid keeping allocator state in // unique_ptr's deleter in ObjectFactory. -class Allocator { +class AllocatorBasic { public: void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); } static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); } @@ -43,5 +42,4 @@ private: GCThreadData& gc_; }; -} // namespace gc -} // namespace kotlin +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocatorTest.cpp similarity index 87% rename from kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocatorTest.cpp index 1e64ff7c66b..d4438cb6f4e 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryAllocatorTest.cpp @@ -3,7 +3,7 @@ * that can be found in the LICENSE file. */ -#include "Allocator.hpp" +#include "ObjectFactoryAllocator.hpp" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -48,7 +48,7 @@ TEST(AllocatorWithGCTest, AllocateWithoutOOM) { EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); EXPECT_CALL(gc, OnOOM(_)).Times(0); } - gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + alloc::AllocatorWithGC allocator(std::move(baseAllocator), gc); void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nonNull); } @@ -64,7 +64,7 @@ TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { EXPECT_CALL(gc, OnOOM(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); } - gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + alloc::AllocatorWithGC allocator(std::move(baseAllocator), gc); void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nonNull); } @@ -79,7 +79,7 @@ TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { EXPECT_CALL(gc, OnOOM(size)); EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); } - gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); + alloc::AllocatorWithGC allocator(std::move(baseAllocator), gc); void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nullptr); } diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp new file mode 100644 index 00000000000..53c4a359930 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweep.hpp @@ -0,0 +1,88 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "FinalizerHooks.hpp" +#include "GC.hpp" +#include "GCStatistics.hpp" +#include "ExtraObjectDataFactory.hpp" + +namespace kotlin::alloc { + +template +void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept { + auto sweepHandle = handle.sweepExtraObjects(); + factoryIter.ApplyDeletions(); + for (auto it = factoryIter.begin(); it != factoryIter.end();) { + auto &extraObject = *it; + if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) { + extraObject.ClearRegularWeakReferenceImpl(); + if (extraObject.HasAssociatedObject()) { + extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); + ++it; + sweepHandle.addKeptObject(); + } else { + extraObject.Uninstall(); + it.EraseAndAdvance(); + sweepHandle.addSweptObject(); + } + } else { + ++it; + sweepHandle.addKeptObject(); + } + } +} + +template +void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept { + auto iter = factory.LockForIter(); + return SweepExtraObjects(handle, iter); +} + +template +typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept { + typename Traits::ObjectFactory::FinalizerQueue finalizerQueue; + auto sweepHandle = handle.sweep(); + + for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) { + if (Traits::TryResetMark(*it)) { + ++it; + sweepHandle.addKeptObject(); + continue; + } + sweepHandle.addSweptObject(); + auto* objHeader = it->GetObjHeader(); + if (HasFinalizers(objHeader)) { + objectFactoryIter.MoveAndAdvance(finalizerQueue, it); + } else { + objectFactoryIter.EraseAndAdvance(it); + } + } + + return finalizerQueue; +} + +template +typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept { + auto iter = objectFactory.LockForIter(); + return Sweep(handle, iter); +} + +template +struct DefaultSweepTraits { + using ObjectFactory = T; + using ExtraObjectsFactory = ExtraObjectDataFactory; + + static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept { + auto* baseObject = object.GetBaseObject(); + if (!baseObject->heap()) return true; + return gc::isMarked(baseObject); + } + + static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); } +}; + +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp similarity index 97% rename from kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp index 2fd398e08f4..531fec30056 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp @@ -3,17 +3,18 @@ * that can be found in the LICENSE file. */ -#include -#include "MarkAndSweepUtils.hpp" +#include "ObjectFactorySweep.hpp" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "Allocator.hpp" -#include "FinalizerHooksTestSupport.hpp" -#include "ObjectFactory.hpp" -#include "ObjectTestSupport.hpp" #include "ExtraObjectDataFactory.hpp" +#include "FinalizerHooksTestSupport.hpp" +#include "MarkAndSweepUtils.hpp" +#include "ObjectFactory.hpp" +#include "ObjectFactoryAllocator.hpp" +#include "ObjectTestSupport.hpp" +#include "TestSupport.hpp" #include "WeakRef.hpp" using namespace kotlin; @@ -45,11 +46,11 @@ struct ObjectFactoryTraits { State state = State::kUnmarked; }; - using Allocator = gc::Allocator; + using Allocator = alloc::AllocatorBasic; }; -using ObjectFactory = mm::ObjectFactory; -using ExtraObjectsDataFactory = mm::ExtraObjectDataFactory; +using ObjectFactory = alloc::ObjectFactory; +using ExtraObjectsDataFactory = alloc::ExtraObjectDataFactory; class Object : public test_support::Object { public: @@ -134,7 +135,7 @@ ObjectFactoryTraits::ObjectData::State GetWeakReferenceState(test_support::Regul struct SweepTraits { using ObjectFactory = ObjectFactory; - using ExtraObjectsFactory = mm::ExtraObjectDataFactory; + using ExtraObjectsFactory = alloc::ExtraObjectDataFactory; static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).ObjectData(); @@ -195,8 +196,8 @@ public: std_support::vector Sweep() { gc::processWeaks(gc::GCHandle::getByEpoch(0), specialRefRegistry_); - gc::SweepExtraObjects(gc::GCHandle::getByEpoch(0), extraObjectFactory_); - auto finalizers = gc::Sweep(gc::GCHandle::getByEpoch(0), objectFactory_); + alloc::SweepExtraObjects(gc::GCHandle::getByEpoch(0), extraObjectFactory_); + auto finalizers = alloc::Sweep(gc::GCHandle::getByEpoch(0), objectFactory_); std_support::vector objects; for (auto node : finalizers.IterForTests()) { objects.push_back(node.GetObjHeader()); @@ -266,7 +267,7 @@ private: kotlin::ScopedMemoryInit memoryInit; FinalizerHooksTestSupport finalizerHooks_; ObjectFactory objectFactory_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gc::Allocator()}; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, alloc::AllocatorBasic()}; ExtraObjectsDataFactory extraObjectFactory_; ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_}; mm::SpecialRefRegistry specialRefRegistry_; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp similarity index 99% rename from kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp index d5519e9ca1e..ba4aad4dc4a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp @@ -11,8 +11,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "Allocator.hpp" #include "FinalizerHooksTestSupport.hpp" +#include "ObjectFactoryAllocator.hpp" #include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" #include "ScopedThread.hpp" @@ -39,7 +39,7 @@ struct DataSizeProvider { }; template -using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; +using ObjectFactoryStorage = alloc::internal::ObjectFactoryStorage; using ObjectFactoryStorageRegular = ObjectFactoryStorage; @@ -848,7 +848,7 @@ public: using Allocator = GlobalMockAllocator; }; -using ObjectFactory = mm::ObjectFactory; +using ObjectFactory = alloc::ObjectFactory; struct Payload { ObjHeader* field1; diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/alloc/mimalloc/cpp/ObjectAlloc.cpp similarity index 78% rename from kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp rename to kotlin-native/runtime/src/alloc/mimalloc/cpp/ObjectAlloc.cpp index 6551b15c23b..c8dd2e9038c 100644 --- a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/alloc/mimalloc/cpp/ObjectAlloc.cpp @@ -7,10 +7,10 @@ #include -#include "../../mimalloc/c/include/mimalloc.h" #include "Alignment.hpp" #include "CompilerConstants.hpp" #include "Memory.h" +#include "mimalloc.h" #if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH #include @@ -28,7 +28,7 @@ std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT; } // namespace -void kotlin::initObjectPool() noexcept { +void alloc::initObjectPool() noexcept { if (!compiler::mimallocUseDefaultOptions()) { std::call_once(initOptions, [] { mi_option_enable(mi_option_reset_decommits); @@ -40,20 +40,20 @@ void kotlin::initObjectPool() noexcept { mi_thread_init(); } -void* kotlin::allocateInObjectPool(size_t size) noexcept { +void* alloc::allocateInObjectPool(size_t size) noexcept { return mi_calloc_aligned(1, size, kObjectAlignment); } -void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept { +void alloc::freeInObjectPool(void* ptr, size_t size) noexcept { mi_free(ptr); } -void kotlin::compactObjectPoolInCurrentThread() noexcept { +void alloc::compactObjectPoolInCurrentThread() noexcept { if (!compiler::mimallocUseCompaction()) return; mi_collect(true); } -void kotlin::compactObjectPoolInMainThread() noexcept { +void alloc::compactObjectPoolInMainThread() noexcept { if (!compiler::mimallocUseCompaction()) return; #if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH if (scheduledCompactOnMainThread.test_and_set()) { @@ -62,14 +62,14 @@ void kotlin::compactObjectPoolInMainThread() noexcept { } dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) { if (mm::IsCurrentThreadRegistered()) { - kotlin::compactObjectPoolInCurrentThread(); + alloc::compactObjectPoolInCurrentThread(); } scheduledCompactOnMainThread.clear(); }); #endif } -size_t kotlin::allocatedBytes() noexcept { +size_t alloc::allocatedBytes() noexcept { return mi_allocated_size(); } diff --git a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/alloc/std/cpp/ObjectAlloc.cpp similarity index 80% rename from kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp rename to kotlin-native/runtime/src/alloc/std/cpp/ObjectAlloc.cpp index dd9b885cbcf..3b42ea53127 100644 --- a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/alloc/std/cpp/ObjectAlloc.cpp @@ -36,9 +36,9 @@ size_t allocatedBytesCounter = 0; } // namespace -void kotlin::initObjectPool() noexcept {} +void alloc::initObjectPool() noexcept {} -void* kotlin::allocateInObjectPool(size_t size) noexcept { +void* alloc::allocateInObjectPool(size_t size) noexcept { // TODO: Check that alignment to kObjectAlignment is satisfied. void* result = callocImpl(1, size); #ifndef KONAN_NO_THREADS @@ -52,7 +52,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept { return result; } -void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept { +void alloc::freeInObjectPool(void* ptr, size_t size) noexcept { #ifndef KONAN_NO_THREADS allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed); #else @@ -61,11 +61,11 @@ void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept { freeImpl(ptr); } -void kotlin::compactObjectPoolInCurrentThread() noexcept {} +void alloc::compactObjectPoolInCurrentThread() noexcept {} -void kotlin::compactObjectPoolInMainThread() noexcept {} +void alloc::compactObjectPoolInMainThread() noexcept {} -size_t kotlin::allocatedBytes() noexcept { +size_t alloc::allocatedBytes() noexcept { #ifndef KONAN_NO_THREADS return allocatedBytesCounter.load(); #else diff --git a/kotlin-native/runtime/src/custom_alloc/README.md b/kotlin-native/runtime/src/custom_alloc/README.md index 6d20c03cc49..0d39bad35f1 100644 --- a/kotlin-native/runtime/src/custom_alloc/README.md +++ b/kotlin-native/runtime/src/custom_alloc/README.md @@ -1,372 +1 @@ -# Overview of the allocator - -This document describes the internals of the custom allocator. The presentation -here is not fully true to the implementation, as the design is still work in -progress. - -The main idea of the custom allocator is to divide system memory into chunks -(pages) that each can be swept independently, in memory consecutive order. -Every allocation ends up as a block of memory inside a page. Each page keeps -track of the size of each block it contains; how this is done depends on the -page type, with different page types optimized for different allocation sizes. -All the memory blocks are consecutive within a page, so the size of a block -also tells where the next block begins. Paired with an additional mechanism -(page type dependent) for determining whether a block is allocated, we can -iterate through the allocated blocks. - -When a thread allocates memory for an object, it will find a page suitable for -the given allocation size. Each thread holds on to a number of pages for the -different size categories. The typical case is that the thread’s current page -for the given size can fit the requested allocation. If that is not the case, a -different page for that size category is requested from the shared allocation -space. The requested page can either be readily available (already prepared by -the GC thread), or it might need to be swept first, or it might be newly -created. - -The GC thread has a new responsibility when using this allocator: While the -mutator threads are paused at the start of GC, the GC thread must prepare the -allocator for sweeping. This does two things. First, it marks all pages as -“needs to be swept before next use”. Second, it releases pages that threads are -holding on to, by clearing the thread local variables for each thread. - -It is possible to have several independent allocation spaces at the same time. -This is currently useful for testing, but is potentially useful in other -settings. - -# Detailed Design - -All allocations are made through a `CustomAllocator` object. - -## [CustomAllocator](cpp/CustomAllocator.hpp) - -```cpp -class CustomAllocator { -public: - CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler); - ObjectHeader* CreateObject(TypeInfo* type); - ArrayHeader* CreateArray(TypeInfo* type, uint32_t count); - ExtraObjectData* CreateExtraObject(); - void PrepareForGc(); - -private: - uint8_t* Allocate(uint64_t cellCount); - uint8_t* AllocateInSingleObjectPage(uint64_t cellCount); - uint8_t* AllocateInNextFitPage(uint32_t cellCount); - uint8_t* AllocateInFixedBlockPage(uint32_t cellCount); - - Heap& heap_; - GCSchedulerThreadData& gcScheduler_; - NextFitPage* nextFitPage_; - FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE]; - ExtraObjectPage* extraObjectPage_; -}; -``` - -The primary responsibility of this class is to delegate each requested -allocation to pages of the appropriate type, based on allocation size. To do -this, it requests pages from the shared allocation space (`Heap`) and stores -pages for later allocations. Each thread thus owns a number of pages for -different allocation sizes, but at most one for each size class. When -allocating, the `CustomAllocator` will first try to allocate in one of its -owned pages. If this fails, it will request a new page for that size class from -a shared `Heap` object. `SingleObjectPages` are never kept by the -`CustomAllocator`, since they are created specifically for a single allocation, -with no extra space. - -## [Heap](cpp/Heap.hpp) - -```cpp -class Heap { -public: - void PrepareForGC(); - - void Sweep(); - - AtomicStack SweepExtraObjects(GCHandle gcHandle); - - FixedBlockPage* GetFixedBlockPage(uint32_t cellCount); - NextFitPage* GetNextFitPage(uint32_t cellCount); - SingleObjectPage* GetSingleObjectPage(uint64_t cellCount); - ExtraObjectPage* GetExtraObjectPage(); - -private: - PageStore fixedBlockPages_[MAX_BLOCK_SIZE]; - PageStore nextFitPages_; - PageStore singleObjectPages_; - AtomicStack extraObjectPages_; - AtomicStack usedExtraObjectPages_; -}; -``` - -A `Heap` object represents a shared allocation space for multiple -`CustomAllocator`s, which can request pages through one of the -`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also -provides a method for sweeping through all blocks that have been allocated in -this heap. The `Heap` object is the synchronization point, and guarantees that -every page is returned at most once. Page ownership is thus implicitly given to -the thread that called the method. The `Heap` object keeps track of all pages, -so there is no need to explicitly return ownership of a page. Internally, a -`Heap` keeps the pages for each size class in a `PageStore`. This means one for -`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes -for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two -`AtomicStack`s, since they require different handling during sweeping. - -## [PageStore](cpp/PageStore.hpp) - -```cpp -template -class PageStore { -public: - void PrepareForGC(); - void Sweep(); - void SweepAndFree(); - PageType* GetPage(uint32_t cellCount); - PageType* NewPage(uint64_t cellCount); - -private: - AtomicStack empty_; - AtomicStack ready_; - AtomicStack used_; - AtomicStack unswept_; -}; -``` - -A PageStore is responsible for keeping track of all pages of a given type and -size class. Each of the pages are in one of four stacks. The stack, that a -given page is in, determines its current state: - -* `unswept_`: have not yet been swept since the last GC cycle. -* `ready_`: are ready for allocation; has been swept by the GC thread. -* `used_`: has been given to some thread for allocation; it might still be used - for allocation, or it might have been discarded with not enough space left. - Will not be used until the next GC cycle. -* `empty_`: same as `ready_`, but does not contain any objects. Will be freed - before the next GC, if not needed before then. - -When a page is requested, the page is taken from `ready_`, if there are any. -Otherwise, an `unswept_` page is taken and swept before returning. If there are -no unswept pages either, an empty page is taken, if there are any. Otherwise a -new page is created in the size category. All returned pages are moved to -`used_`. During the marking phase, all remaining pages in `empty_` are freed, -and all other pages are moved to `unswept_`. The GC thread will go through all -`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one -of the other threads sweeps a page from `unswept_`, it is moved directly to -`used_`, as it is claimed by the `CustomAllocator` that swept it. - -`SingleObjectPage`s are treated slightly differently, because they are created -for one specific single allocation, and not reused when that allocation is -freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without -checking any of the stacks, and during sweeping, they are freed directly rather -than being put into the `empty_` stack. Ideally, there would only be two stacks -in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little -is lost by just using the existing `PageStore` logic used for the other pages. - -## [AtomicStack](cpp/AtomicStack.hpp) - -```cpp -template -class AtomicStack { -public: - PageType* Pop(); - void Push(PageType* elm); - void TransferAllFrom(AtomicStack& src); - bool isEmpty(); - - -private: - std::atomic stack_; -}; -``` - -The only place where atomics are used are in the stacks inside the `PageStore`. -All page classes have a non-atomic next pointer, to be used for linking up in -exactly one stack. `Pop` and `Push` are implemented with compare-and-swap -operations. The class is thread safe, except for if an element is freed while -another thread tries to Pop it from a stack. - -# Page types - -This section is likely to change, given the likely introduction of additional -page types. It also describes some details about which page type is chosen for -a given allocation, which is also likely to change. - -There are four different page types, but they all share the feature that they -can be swept independently. The Sweep methods return whether there were any -live objects in the page after sweeping. If not, the page will be given back to -the OS. - -## [FixedBlockPage](cpp/FixedBlockPage.hpp) - -```cpp -class FixedBlockPage { -public: - FixedBlockPage(uint32_t blockSize) noexcept; - - uint8_t* TryAllocate() noexcept; - - bool Sweep() noexcept; - -private: - FixedBlockPage* next_; - FixedCellRange nextFree_; - uint32_t blockSize_; - uint32_t end_; - FixedBlockCell cells_[]; -}; -}; -``` - -All sufficiently small allocations (currently arbitrary <1KiB) are directed to -a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations -are expected to be in this page type. A `FixedBlockPage` consists of a number -of equally sized blocks, where each allocation will take up exactly one such -block. - -```cpp -struct FixedBlockCell { - union { - uint8_t data[]; - FixedCellRange nextFree; - } -}; - -struct alignas(8) FixedCellRange { - uint32_t first; - uint32_t last; -}; -``` - -Consecutive unallocated cells are represented by a `FixedCellRange`, with -`.first` and `.last` being the inclusive end points of the range of unallocated -cells. The `FixedBlockCell` at the the `.last` index contains a -`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange` -of unallocated ranges thus form a linked list. - -The important point is that all links in this list point forward in the page, so -all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a -`FixedBlockPage` consists of walking the free-list forward, and sweeping all -blocks in between the links, maintaining the free list when blocks are freed. - -Each small page takes up the same amount of space, independent of block size, -so larger block size implies fewer blocks per page. This size is arbitrarily -chosen to be 256 KiB, but this might change. - -## [NextFitPage](cpp/NextFitPage.hpp) - -```cpp -class NextFitPage { -public: - NextFitPage(uint32_t cellCount); - Cell* TryAllocate(uint32_t cellCount); - bool Sweep(); - -private: - NextFitPage* next_; // used by AtomicStack - Cell* curBlock_; - Cell cells_[]; -}; -``` - -Allocations that could theoretically fit in a `FixedBlockPage`, but would -require too large a block size (arbitrary >=1KiB), are allocated in a -`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64 -KiB for experiments). All blocks in a `NextFitPage` have a header that tells how -big the block is, and whether it is allocated or not. There are no gaps between -blocks, so the size of a block also tells where the next block is. The header -information fits inside a 8 byte `Cell`. - -```cpp -class Cell { -public: - Cell(uint32_t size); - uint8_t* TryAllocate(uint32_t cellCount); - -private: - uint32_t isAllocated_; - uint32_t size_; - uint8_t data_[]; -}; -``` - -The page keeps a reference to a currently active block, and will try to bump -allocate inside that block. If allocation does not fit, we move to the next -block that fits. If no block in the page fits the requested size, the page is -abandoned until the next GC. - -## [SingleObjectPage](cpp/SingleObjectPage.hpp) - -```cpp -class SingleObjectPage { -public: - SingleObjectPage(uint64_t cellCount); - bool Sweep(); - -private: - SingleObjectPage* next_; // used by AtomicStack -}; -``` - -Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`, -which only contains that single block of the requested size. They are also -handled slightly differently by both `Heap` and `CustomAllocator`. First off, -`Heap::GetSingleObjectPage` will never check existing pages, and instead just -allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to -any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC -thread. - -## [ExtraObjectPage](cpp/ExtraObjectPage.hpp) - -```cpp -class ExtraObjectPage { -public: - ExtraObjectPage(); - ExtraObjectData* TryAllocate(); - bool Sweep(FinalizerQueue& queue); - - -private: - ExtraObjectPage* next_; // used by AtomicStack - ExtraObjectCell* nextFree_; - ExtraObjectCell cells_[]; -}; -``` - -Extra objects are used for attaching additional data to some objects. This is -used for objects that require special handling during garbage collection: - -* objects with finalizers -* weak references -* interop references - -Extra objects are allocated in `ExtraObjectPage`s, which are very similar to -`FixedBlockPage`s. They primarily differ in how they are swept, since it is -during sweeping of `ExtraObject`s that scheduling of finalization happens. If -an object that requires finalization is found, it is added to the -`FinalizerQueue` given as argument. The cells are also slightly different, in -that they add a new field that allows the cells to be added to the finalizer -queue. - -```cpp -struct ExtraObjectCell { - ExtraObjectCell* next_; // used by AtomicStack - ExtraObjectData data_; -}; -``` - -# Finalizers - -Section like to change. - -In the existing memory model, finalization tasks are found and scheduled during -sweeping of regular objects. The objects to be finalized are chained together -using a pointer in the Node header, added to all allocated objects. This header -is not needed in the custom allocator, apart from linking in the finalization -queue. - -We therefore reintroduce this pointer in a header for `ExtraObjectData`. For -this, we reuse the `ExtraObjectCell` as header for both free list pointer and as -linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`. - -# Enabling the allocator - -The custom allocator is enabled with the compiler flag -Xallocator=custom. +Allocator documentation was moved [here](../alloc/custom/README.md). diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp deleted file mode 100644 index c39f2fb8ab2..00000000000 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "CustomLogging.hpp" - -#include "GCApi.hpp" - -// These functions are just stubs to make the existing object creation -// infrastructure link correctly, but if they are ever called, something went -// wrong. - -namespace kotlin { - -void* allocateInObjectPool(size_t size) noexcept { - CustomAllocWarning("static allocateInObjectPool(%zu) not supported", size); - return nullptr; -} - -void freeInObjectPool(void* ptr, size_t size) noexcept { - CustomAllocWarning("static freeInObjectPool(%p, %zu) not supported", ptr, size); -} - -void initObjectPool() noexcept {} -void compactObjectPoolInMainThread() noexcept {} -void compactObjectPoolInCurrentThread() noexcept {} - -size_t allocatedBytes() noexcept { - return alloc::GetAllocatedBytes(); -} - -} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp index e8660e8027d..a5b33896534 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp @@ -7,6 +7,8 @@ #ifdef CUSTOM_ALLOCATOR +// TODO: Move into alloc/custom/AllocatorImpl.hpp + #include "CustomAllocator.hpp" #include "CustomFinalizerProcessor.hpp" #include "GCApi.hpp" @@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits; #else -#include "Allocator.hpp" +// TODO: Move into alloc/legacy/AllocatorImpl.hpp + #include "ExtraObjectDataFactory.hpp" #include "GC.hpp" #include "GlobalData.hpp" #include "Logging.hpp" #include "ObjectFactory.hpp" +#include "ObjectFactoryAllocator.hpp" +#include "ObjectFactorySweep.hpp" namespace kotlin::gc { struct ObjectFactoryTraits { - using Allocator = AllocatorWithGC; + using Allocator = alloc::AllocatorWithGC; using ObjectData = gc::GC::ObjectData; - Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } + Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); } void OnOOM(size_t size) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); @@ -51,7 +56,7 @@ struct ObjectFactoryTraits { } }; -using ObjectFactory = mm::ObjectFactory; +using ObjectFactory = alloc::ObjectFactory; inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { return ObjectFactory::NodeRef::From(object).ObjectData(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 6ad9fcc1297..a025b35d705 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -92,7 +92,7 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const #ifndef CUSTOM_ALLOCATOR gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( ObjectFactory& objectFactory, - mm::ExtraObjectDataFactory& extraObjectDataFactory, + alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept : @@ -211,11 +211,11 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { resumeTheWorld(gcHandle); #ifndef CUSTOM_ALLOCATOR - gc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); + alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = gc::Sweep>(gcHandle, *objectFactoryIterable); + auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; - kotlin::compactObjectPoolInMainThread(); + alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects auto finalizerQueue = heap_.Sweep(gcHandle); @@ -224,7 +224,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { } finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, allocatedBytes()); + scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index c002150014e..91ab8a55a9b 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -67,7 +67,7 @@ public: #else ConcurrentMarkAndSweep( ObjectFactory& objectFactory, - mm::ExtraObjectDataFactory& extraObjectDataFactory, + alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& scheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; @@ -93,7 +93,7 @@ private: #ifndef CUSTOM_ALLOCATOR ObjectFactory& objectFactory_; - mm::ExtraObjectDataFactory& extraObjectDataFactory_; + alloc::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; #endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 4308caa424e..475952e07e2 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -9,7 +9,6 @@ #include "GC.hpp" #include "GCStatistics.hpp" #include "MarkAndSweepUtils.hpp" -#include "ObjectAlloc.hpp" #include "ObjectOps.hpp" #include "std_support/Memory.hpp" @@ -100,7 +99,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { } size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { - return allocatedBytes(); +#ifdef CUSTOM_ALLOCATOR + return alloc::GetAllocatedBytes(); +#else + return alloc::allocatedBytes(); +#endif } void gc::GC::ClearForTests() noexcept { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 887d4007a8e..a197708d6b2 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -23,14 +23,14 @@ public: #ifndef CUSTOM_ALLOCATOR ObjectFactory& objectFactory() noexcept { return objectFactory_; } - mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } + alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif ConcurrentMarkAndSweep& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR ObjectFactory objectFactory_; - mm::ExtraObjectDataFactory extraObjectDataFactory_; + alloc::ExtraObjectDataFactory extraObjectDataFactory_; #endif ConcurrentMarkAndSweep gc_; }; @@ -53,7 +53,7 @@ public: alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } + alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: @@ -63,7 +63,7 @@ private: #else [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; + alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index b96b223056f..f95b1d94878 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -7,7 +7,6 @@ #define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H #include "ExtraObjectData.hpp" -#include "ExtraObjectDataFactory.hpp" #include "FinalizerHooks.hpp" #include "GlobalData.hpp" #include "GCStatistics.hpp" @@ -106,79 +105,6 @@ void Mark(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQue } } -template -void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept { - auto sweepHandle = handle.sweepExtraObjects(); - factoryIter.ApplyDeletions(); - for (auto it = factoryIter.begin(); it != factoryIter.end();) { - auto &extraObject = *it; - if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) { - extraObject.ClearRegularWeakReferenceImpl(); - if (extraObject.HasAssociatedObject()) { - extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); - ++it; - sweepHandle.addKeptObject(); - } else { - extraObject.Uninstall(); - it.EraseAndAdvance(); - sweepHandle.addSweptObject(); - } - } else { - ++it; - sweepHandle.addKeptObject(); - } - } -} - -template -void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept { - auto iter = factory.LockForIter(); - return SweepExtraObjects(handle, iter); -} - -template -typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept { - typename Traits::ObjectFactory::FinalizerQueue finalizerQueue; - auto sweepHandle = handle.sweep(); - - for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) { - if (Traits::TryResetMark(*it)) { - ++it; - sweepHandle.addKeptObject(); - continue; - } - sweepHandle.addSweptObject(); - auto* objHeader = it->GetObjHeader(); - if (HasFinalizers(objHeader)) { - objectFactoryIter.MoveAndAdvance(finalizerQueue, it); - } else { - objectFactoryIter.EraseAndAdvance(it); - } - } - - return finalizerQueue; -} - -template -typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept { - auto iter = objectFactory.LockForIter(); - return Sweep(handle, iter); -} - -template -struct DefaultSweepTraits { - using ObjectFactory = T; - using ExtraObjectsFactory = mm::ExtraObjectDataFactory; - - static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept { - auto* baseObject = object.GetBaseObject(); - if (!baseObject->heap()) return true; - return gc::isMarked(baseObject); - } - - static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); } -}; - template void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { auto handle = gcHandle.collectThreadRoots(thread); diff --git a/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp index ab53ce6c561..01f5544e460 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp @@ -7,26 +7,31 @@ #ifdef CUSTOM_ALLOCATOR +// TODO: Move into alloc/custom/AllocatorImpl.hpp + #include "CustomAllocator.hpp" +#include "GCApi.hpp" #include "Heap.hpp" #else -#include "Allocator.hpp" +// TODO: Move into alloc/legacy/AllocatorImpl.hpp + #include "ExtraObjectDataFactory.hpp" #include "GC.hpp" #include "ObjectFactory.hpp" +#include "ObjectFactoryAllocator.hpp" namespace kotlin::gc { struct ObjectFactoryTraits { - using Allocator = Allocator; + using Allocator = alloc::AllocatorBasic; using ObjectData = gc::GC::ObjectData; Allocator CreateAllocator() noexcept { return Allocator(); } }; -using ObjectFactory = mm::ObjectFactory; +using ObjectFactory = alloc::ObjectFactory; } // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index ae33fdd2d96..d316c279423 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -9,7 +9,6 @@ #include "GC.hpp" #include "GCStatistics.hpp" #include "NoOpGC.hpp" -#include "ObjectAlloc.hpp" #include "ObjectOps.hpp" #include "ThreadData.hpp" #include "std_support/Memory.hpp" @@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { } size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { - return allocatedBytes(); +#ifdef CUSTOM_ALLOCATOR + return alloc::GetAllocatedBytes(); +#else + return alloc::allocatedBytes(); +#endif } void gc::GC::ClearForTests() noexcept { diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 64f441f6968..7ffbbf35995 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -19,14 +19,14 @@ public: #ifndef CUSTOM_ALLOCATOR ObjectFactory& objectFactory() noexcept { return objectFactory_; } - mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } + alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif NoOpGC& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR ObjectFactory objectFactory_; - mm::ExtraObjectDataFactory extraObjectDataFactory_; + alloc::ExtraObjectDataFactory extraObjectDataFactory_; #endif NoOpGC gc_; }; @@ -46,7 +46,7 @@ public: alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } + alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: @@ -56,7 +56,7 @@ private: #else [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; + alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp index e8660e8027d..a5b33896534 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp @@ -7,6 +7,8 @@ #ifdef CUSTOM_ALLOCATOR +// TODO: Move into alloc/custom/AllocatorImpl.hpp + #include "CustomAllocator.hpp" #include "CustomFinalizerProcessor.hpp" #include "GCApi.hpp" @@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits; #else -#include "Allocator.hpp" +// TODO: Move into alloc/legacy/AllocatorImpl.hpp + #include "ExtraObjectDataFactory.hpp" #include "GC.hpp" #include "GlobalData.hpp" #include "Logging.hpp" #include "ObjectFactory.hpp" +#include "ObjectFactoryAllocator.hpp" +#include "ObjectFactorySweep.hpp" namespace kotlin::gc { struct ObjectFactoryTraits { - using Allocator = AllocatorWithGC; + using Allocator = alloc::AllocatorWithGC; using ObjectData = gc::GC::ObjectData; - Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } + Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); } void OnOOM(size_t size) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); @@ -51,7 +56,7 @@ struct ObjectFactoryTraits { } }; -using ObjectFactory = mm::ObjectFactory; +using ObjectFactory = alloc::ObjectFactory; inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { return ObjectFactory::NodeRef::From(object).ObjectData(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 8f7deb2d586..1227758ed8c 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -9,7 +9,6 @@ #include "GCStatistics.hpp" #include "GlobalData.hpp" #include "MarkAndSweepUtils.hpp" -#include "ObjectAlloc.hpp" #include "ObjectOps.hpp" #include "SameThreadMarkAndSweep.hpp" #include "std_support/Memory.hpp" @@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { } size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { - return allocatedBytes(); +#ifdef CUSTOM_ALLOCATOR + return alloc::GetAllocatedBytes(); +#else + return alloc::allocatedBytes(); +#endif } void gc::GC::ClearForTests() noexcept { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index 9e54d8e38b6..638db32f43a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -21,14 +21,14 @@ public: explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} ObjectFactory& objectFactory() noexcept { return objectFactory_; } - mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } + alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif SameThreadMarkAndSweep& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR ObjectFactory objectFactory_; - mm::ExtraObjectDataFactory extraObjectDataFactory_; + alloc::ExtraObjectDataFactory extraObjectDataFactory_; #endif SameThreadMarkAndSweep gc_; }; @@ -49,7 +49,7 @@ public: SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; } #ifndef CUSTOM_ALLOCATOR ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } + alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #else alloc::CustomAllocator& alloc() noexcept { return alloc_; } #endif @@ -61,7 +61,7 @@ private: #else [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; + alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 5664846196e..913003a465d 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -14,7 +14,6 @@ #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" #include "Memory.h" -#include "ObjectAlloc.hpp" #include "RootSet.hpp" #include "Runtime.h" #include "ThreadData.hpp" @@ -28,7 +27,7 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( gcScheduler::GCScheduler& gcScheduler) noexcept : #else gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : + ObjectFactory& objectFactory, alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), extraObjectDataFactory_(extraObjectDataFactory), @@ -103,11 +102,11 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); std::optional objectFactoryIterable = objectFactory_.LockForIter(); - gc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); + alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = gc::Sweep>(gcHandle, *objectFactoryIterable); + auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; - kotlin::compactObjectPoolInMainThread(); + alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects auto finalizerQueue = heap_.Sweep(gcHandle); @@ -117,7 +116,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, allocatedBytes()); + scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes()); resumeTheWorld(gcHandle); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 03755be9550..761201e6ab3 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -45,7 +45,7 @@ public: #else SameThreadMarkAndSweep( ObjectFactory& objectFactory, - mm::ExtraObjectDataFactory& extraObjectDataFactory, + alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept; #endif @@ -66,7 +66,7 @@ private: #ifndef CUSTOM_ALLOCATOR ObjectFactory& objectFactory_; - mm::ExtraObjectDataFactory& extraObjectDataFactory_; + alloc::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; #endif diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index b45177c5881..328ad10eea3 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -612,6 +612,9 @@ bool FinalizersThreadIsRunning() noexcept; void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept; +void initObjectPool() noexcept; +void compactObjectPoolInCurrentThread() noexcept; + } // namespace kotlin RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object); diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp index de526479438..a181f737ccc 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp +++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp @@ -11,7 +11,6 @@ #include "MainQueueProcessor.hpp" #include "Memory.h" #include "ObjCExportInit.h" -#include "ObjectAlloc.hpp" #include "Porting.h" #include "Runtime.h" #include "RuntimePrivate.hpp" diff --git a/kotlin-native/runtime/src/main/cpp/Worker.cpp b/kotlin-native/runtime/src/main/cpp/Worker.cpp index cc0ce802ee3..e6e2384d781 100644 --- a/kotlin-native/runtime/src/main/cpp/Worker.cpp +++ b/kotlin-native/runtime/src/main/cpp/Worker.cpp @@ -32,7 +32,6 @@ #include "Memory.h" #include "Natives.h" #include "ObjCMMAPI.h" -#include "ObjectAlloc.hpp" #include "Runtime.h" #include "Types.h" #include "Worker.h" diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index d4aa2638787..08edc382f93 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -6,6 +6,7 @@ #include "Memory.h" #include "MemoryPrivate.hpp" +#include "Allocator.hpp" #include "Exceptions.h" #include "ExtraObjectData.hpp" #include "Freezing.hpp" @@ -636,3 +637,11 @@ RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakR void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept { mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes); } + +void kotlin::initObjectPool() noexcept { + alloc::initObjectPool(); +} + +void kotlin::compactObjectPoolInCurrentThread() noexcept { + alloc::compactObjectPoolInCurrentThread(); +}