[K/N] Refactor allocator modules ^KT-60928
- Move std_alloc, opt_alloc and custom_alloc into alloc/std, alloc/mimalloc and alloc/custom to mirror convention of gc, gcScheduler modules. - Add alloc/common with common allocator API - Add alloc/legacy with common implementation details of alloc/std and alloc/mimalloc. alloc/custom does not depend on alloc/legacy. - Removes experimental_memory_manager_custom as it's now unused - Additionally, renames experimental_memory_manager module into mm
This commit is contained in:
Alexander Shabalin
committed by
Space Team
Space Team
parent
c6893de9bb
commit
1fa0ef6f56
+5
-3
@@ -298,6 +298,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
||||
|
||||
internal val runtimeNativeLibraries: List<String> = mutableListOf<String>().apply {
|
||||
if (debug) add("debug.bc")
|
||||
add("mm.bc")
|
||||
add("common_alloc.bc")
|
||||
add("common_gc.bc")
|
||||
add("common_gcScheduler.bc")
|
||||
when (gcSchedulerType) {
|
||||
@@ -315,14 +317,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
||||
}
|
||||
}
|
||||
if (allocationMode == AllocationMode.CUSTOM) {
|
||||
add("experimental_memory_manager_custom.bc")
|
||||
when (gc) {
|
||||
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc")
|
||||
GC.NOOP -> add("noop_gc_custom.bc")
|
||||
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc")
|
||||
}
|
||||
} else {
|
||||
add("experimental_memory_manager.bc")
|
||||
when (gc) {
|
||||
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc")
|
||||
GC.NOOP -> add("noop_gc.bc")
|
||||
@@ -339,10 +339,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
||||
}
|
||||
when (allocationMode) {
|
||||
AllocationMode.MIMALLOC -> {
|
||||
add("opt_alloc.bc")
|
||||
add("legacy_alloc.bc")
|
||||
add("mimalloc_alloc.bc")
|
||||
add("mimalloc.bc")
|
||||
}
|
||||
AllocationMode.STD -> {
|
||||
add("legacy_alloc.bc")
|
||||
add("std_alloc.bc")
|
||||
}
|
||||
AllocationMode.CUSTOM -> {
|
||||
|
||||
@@ -150,15 +150,25 @@ bitcode {
|
||||
}
|
||||
}
|
||||
|
||||
module("common_alloc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/alloc/common"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
}
|
||||
}
|
||||
|
||||
module("std_alloc") {
|
||||
headersDirs.from(files("src/main/cpp"))
|
||||
srcRoot.set(layout.projectDirectory.dir("src/alloc/std"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
}
|
||||
}
|
||||
|
||||
module("custom_alloc") {
|
||||
headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp"))
|
||||
srcRoot.set(layout.projectDirectory.dir("src/alloc/custom"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
@@ -167,8 +177,9 @@ bitcode {
|
||||
compilerArgs.add("-DCUSTOM_ALLOCATOR")
|
||||
}
|
||||
|
||||
module("opt_alloc") {
|
||||
headersDirs.from(files("src/main/cpp"))
|
||||
module("mimalloc_alloc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/alloc/mimalloc"))
|
||||
headersDirs.from(files("src/mimalloc/c/include", "src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
}
|
||||
@@ -176,6 +187,15 @@ bitcode {
|
||||
compilerArgs.add("-DKONAN_MI_MALLOC=1")
|
||||
}
|
||||
|
||||
module("legacy_alloc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/alloc/legacy"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
}
|
||||
}
|
||||
|
||||
module("exceptionsSupport") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/exceptions_support"))
|
||||
headersDirs.from(files("src/main/cpp"))
|
||||
@@ -227,9 +247,8 @@ bitcode {
|
||||
}
|
||||
}
|
||||
|
||||
module("experimental_memory_manager") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/mm"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp"))
|
||||
module("mm") {
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -237,21 +256,9 @@ bitcode {
|
||||
}
|
||||
}
|
||||
|
||||
module("experimental_memory_manager_custom") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/mm"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
test {}
|
||||
}
|
||||
|
||||
compilerArgs.add("-DCUSTOM_ALLOCATOR")
|
||||
}
|
||||
|
||||
module("common_gc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/common"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
@@ -260,7 +267,7 @@ bitcode {
|
||||
|
||||
module("noop_gc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -269,7 +276,7 @@ bitcode {
|
||||
|
||||
module("noop_gc_custom") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -280,7 +287,7 @@ bitcode {
|
||||
|
||||
module("same_thread_ms_gc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -290,7 +297,7 @@ bitcode {
|
||||
|
||||
module("same_thread_ms_gc_custom") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -302,7 +309,7 @@ bitcode {
|
||||
|
||||
module("concurrent_ms_gc") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -312,7 +319,7 @@ bitcode {
|
||||
|
||||
module("concurrent_ms_gc_custom") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
testFixtures {}
|
||||
@@ -324,7 +331,7 @@ bitcode {
|
||||
|
||||
module("common_gcScheduler") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/common"))
|
||||
headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
@@ -333,7 +340,7 @@ bitcode {
|
||||
|
||||
module("manual_gcScheduler") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/manual"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
}
|
||||
@@ -341,7 +348,7 @@ bitcode {
|
||||
|
||||
module("adaptive_gcScheduler") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/adaptive"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
@@ -350,7 +357,7 @@ bitcode {
|
||||
|
||||
module("aggressive_gcScheduler") {
|
||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/aggressive"))
|
||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||
sourceSets {
|
||||
main {}
|
||||
test {}
|
||||
@@ -359,56 +366,56 @@ bitcode {
|
||||
|
||||
testsGroup("custom_alloc_runtime_tests") {
|
||||
testedModules.addAll("custom_alloc")
|
||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "objc")
|
||||
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc_custom", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_mimalloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "opt_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_std_alloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_custom_alloc_runtime_tests") {
|
||||
testedModules.addAll("experimental_memory_manager_custom", "same_thread_ms_gc_custom")
|
||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
testedModules.addAll("mm", "same_thread_ms_gc_custom")
|
||||
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_cms_mimalloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_cms_std_alloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") {
|
||||
testedModules.addAll("experimental_memory_manager_custom", "concurrent_ms_gc_custom")
|
||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
testedModules.addAll("mm", "concurrent_ms_gc_custom")
|
||||
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_noop_mimalloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_noop_std_alloc_runtime_tests") {
|
||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc")
|
||||
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") {
|
||||
testedModules.addAll("experimental_memory_manager_custom", "noop_gc_custom")
|
||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
testedModules.addAll("mm", "noop_gc_custom")
|
||||
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("aggressive_gcScheduler_runtime_tests") {
|
||||
testedModules.addAll("aggressive_gcScheduler")
|
||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc")
|
||||
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
|
||||
testsGroup("adaptive_gcScheduler_runtime_tests") {
|
||||
testedModules.addAll("adaptive_gcScheduler")
|
||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc")
|
||||
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
/*
|
||||
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include "Allocator.hpp"
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace kotlin::alloc {
|
||||
|
||||
// TODO: Build `Allocator`, `Allocator::ThreadData` like with `gc`, `gcScheduler`,
|
||||
// and move allocator-specific data there.
|
||||
|
||||
void initObjectPool() noexcept;
|
||||
void compactObjectPoolInCurrentThread() noexcept;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
# Overview of the allocator
|
||||
|
||||
This document describes the internals of the custom allocator. The presentation
|
||||
here is not fully true to the implementation, as the design is still work in
|
||||
progress.
|
||||
|
||||
The main idea of the custom allocator is to divide system memory into chunks
|
||||
(pages) that each can be swept independently, in memory consecutive order.
|
||||
Every allocation ends up as a block of memory inside a page. Each page keeps
|
||||
track of the size of each block it contains; how this is done depends on the
|
||||
page type, with different page types optimized for different allocation sizes.
|
||||
All the memory blocks are consecutive within a page, so the size of a block
|
||||
also tells where the next block begins. Paired with an additional mechanism
|
||||
(page type dependent) for determining whether a block is allocated, we can
|
||||
iterate through the allocated blocks.
|
||||
|
||||
When a thread allocates memory for an object, it will find a page suitable for
|
||||
the given allocation size. Each thread holds on to a number of pages for the
|
||||
different size categories. The typical case is that the thread’s current page
|
||||
for the given size can fit the requested allocation. If that is not the case, a
|
||||
different page for that size category is requested from the shared allocation
|
||||
space. The requested page can either be readily available (already prepared by
|
||||
the GC thread), or it might need to be swept first, or it might be newly
|
||||
created.
|
||||
|
||||
The GC thread has a new responsibility when using this allocator: While the
|
||||
mutator threads are paused at the start of GC, the GC thread must prepare the
|
||||
allocator for sweeping. This does two things. First, it marks all pages as
|
||||
“needs to be swept before next use”. Second, it releases pages that threads are
|
||||
holding on to, by clearing the thread local variables for each thread.
|
||||
|
||||
It is possible to have several independent allocation spaces at the same time.
|
||||
This is currently useful for testing, but is potentially useful in other
|
||||
settings.
|
||||
|
||||
# Detailed Design
|
||||
|
||||
All allocations are made through a `CustomAllocator` object.
|
||||
|
||||
## [CustomAllocator](cpp/CustomAllocator.hpp)
|
||||
|
||||
```cpp
|
||||
class CustomAllocator {
|
||||
public:
|
||||
CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler);
|
||||
ObjectHeader* CreateObject(TypeInfo* type);
|
||||
ArrayHeader* CreateArray(TypeInfo* type, uint32_t count);
|
||||
ExtraObjectData* CreateExtraObject();
|
||||
void PrepareForGc();
|
||||
|
||||
private:
|
||||
uint8_t* Allocate(uint64_t cellCount);
|
||||
uint8_t* AllocateInSingleObjectPage(uint64_t cellCount);
|
||||
uint8_t* AllocateInNextFitPage(uint32_t cellCount);
|
||||
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount);
|
||||
|
||||
Heap& heap_;
|
||||
GCSchedulerThreadData& gcScheduler_;
|
||||
NextFitPage* nextFitPage_;
|
||||
FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||
ExtraObjectPage* extraObjectPage_;
|
||||
};
|
||||
```
|
||||
|
||||
The primary responsibility of this class is to delegate each requested
|
||||
allocation to pages of the appropriate type, based on allocation size. To do
|
||||
this, it requests pages from the shared allocation space (`Heap`) and stores
|
||||
pages for later allocations. Each thread thus owns a number of pages for
|
||||
different allocation sizes, but at most one for each size class. When
|
||||
allocating, the `CustomAllocator` will first try to allocate in one of its
|
||||
owned pages. If this fails, it will request a new page for that size class from
|
||||
a shared `Heap` object. `SingleObjectPages` are never kept by the
|
||||
`CustomAllocator`, since they are created specifically for a single allocation,
|
||||
with no extra space.
|
||||
|
||||
## [Heap](cpp/Heap.hpp)
|
||||
|
||||
```cpp
|
||||
class Heap {
|
||||
public:
|
||||
void PrepareForGC();
|
||||
|
||||
void Sweep();
|
||||
|
||||
AtomicStack<ExtraObjectCell> SweepExtraObjects(GCHandle gcHandle);
|
||||
|
||||
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount);
|
||||
NextFitPage* GetNextFitPage(uint32_t cellCount);
|
||||
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount);
|
||||
ExtraObjectPage* GetExtraObjectPage();
|
||||
|
||||
private:
|
||||
PageStore<FixedBlockPage> fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||
PageStore<NextFitPage> nextFitPages_;
|
||||
PageStore<SingleObjectPage> singleObjectPages_;
|
||||
AtomicStack<ExtraObjectPage> extraObjectPages_;
|
||||
AtomicStack<ExtraObjectPage> usedExtraObjectPages_;
|
||||
};
|
||||
```
|
||||
|
||||
A `Heap` object represents a shared allocation space for multiple
|
||||
`CustomAllocator`s, which can request pages through one of the
|
||||
`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also
|
||||
provides a method for sweeping through all blocks that have been allocated in
|
||||
this heap. The `Heap` object is the synchronization point, and guarantees that
|
||||
every page is returned at most once. Page ownership is thus implicitly given to
|
||||
the thread that called the method. The `Heap` object keeps track of all pages,
|
||||
so there is no need to explicitly return ownership of a page. Internally, a
|
||||
`Heap` keeps the pages for each size class in a `PageStore`. This means one for
|
||||
`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes
|
||||
for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two
|
||||
`AtomicStack`s, since they require different handling during sweeping.
|
||||
|
||||
## [PageStore](cpp/PageStore.hpp)
|
||||
|
||||
```cpp
|
||||
template <class PageType>
|
||||
class PageStore {
|
||||
public:
|
||||
void PrepareForGC();
|
||||
void Sweep();
|
||||
void SweepAndFree();
|
||||
PageType* GetPage(uint32_t cellCount);
|
||||
PageType* NewPage(uint64_t cellCount);
|
||||
|
||||
private:
|
||||
AtomicStack<PageType> empty_;
|
||||
AtomicStack<PageType> ready_;
|
||||
AtomicStack<PageType> used_;
|
||||
AtomicStack<PageType> unswept_;
|
||||
};
|
||||
```
|
||||
|
||||
A PageStore is responsible for keeping track of all pages of a given type and
|
||||
size class. Each of the pages are in one of four stacks. The stack, that a
|
||||
given page is in, determines its current state:
|
||||
|
||||
* `unswept_`: have not yet been swept since the last GC cycle.
|
||||
* `ready_`: are ready for allocation; has been swept by the GC thread.
|
||||
* `used_`: has been given to some thread for allocation; it might still be used
|
||||
for allocation, or it might have been discarded with not enough space left.
|
||||
Will not be used until the next GC cycle.
|
||||
* `empty_`: same as `ready_`, but does not contain any objects. Will be freed
|
||||
before the next GC, if not needed before then.
|
||||
|
||||
When a page is requested, the page is taken from `ready_`, if there are any.
|
||||
Otherwise, an `unswept_` page is taken and swept before returning. If there are
|
||||
no unswept pages either, an empty page is taken, if there are any. Otherwise a
|
||||
new page is created in the size category. All returned pages are moved to
|
||||
`used_`. During the marking phase, all remaining pages in `empty_` are freed,
|
||||
and all other pages are moved to `unswept_`. The GC thread will go through all
|
||||
`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one
|
||||
of the other threads sweeps a page from `unswept_`, it is moved directly to
|
||||
`used_`, as it is claimed by the `CustomAllocator` that swept it.
|
||||
|
||||
`SingleObjectPage`s are treated slightly differently, because they are created
|
||||
for one specific single allocation, and not reused when that allocation is
|
||||
freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without
|
||||
checking any of the stacks, and during sweeping, they are freed directly rather
|
||||
than being put into the `empty_` stack. Ideally, there would only be two stacks
|
||||
in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little
|
||||
is lost by just using the existing `PageStore` logic used for the other pages.
|
||||
|
||||
## [AtomicStack](cpp/AtomicStack.hpp)
|
||||
|
||||
```cpp
|
||||
template <class PageType>
|
||||
class AtomicStack {
|
||||
public:
|
||||
PageType* Pop();
|
||||
void Push(PageType* elm);
|
||||
void TransferAllFrom(AtomicStack<T>& src);
|
||||
bool isEmpty();
|
||||
|
||||
|
||||
private:
|
||||
std::atomic<PageType*> stack_;
|
||||
};
|
||||
```
|
||||
|
||||
The only place where atomics are used are in the stacks inside the `PageStore`.
|
||||
All page classes have a non-atomic next pointer, to be used for linking up in
|
||||
exactly one stack. `Pop` and `Push` are implemented with compare-and-swap
|
||||
operations. The class is thread safe, except for if an element is freed while
|
||||
another thread tries to Pop it from a stack.
|
||||
|
||||
# Page types
|
||||
|
||||
This section is likely to change, given the likely introduction of additional
|
||||
page types. It also describes some details about which page type is chosen for
|
||||
a given allocation, which is also likely to change.
|
||||
|
||||
There are four different page types, but they all share the feature that they
|
||||
can be swept independently. The Sweep methods return whether there were any
|
||||
live objects in the page after sweeping. If not, the page will be given back to
|
||||
the OS.
|
||||
|
||||
## [FixedBlockPage](cpp/FixedBlockPage.hpp)
|
||||
|
||||
```cpp
|
||||
class FixedBlockPage {
|
||||
public:
|
||||
FixedBlockPage(uint32_t blockSize) noexcept;
|
||||
|
||||
uint8_t* TryAllocate() noexcept;
|
||||
|
||||
bool Sweep() noexcept;
|
||||
|
||||
private:
|
||||
FixedBlockPage* next_;
|
||||
FixedCellRange nextFree_;
|
||||
uint32_t blockSize_;
|
||||
uint32_t end_;
|
||||
FixedBlockCell cells_[];
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
All sufficiently small allocations (currently arbitrary <1KiB) are directed to
|
||||
a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations
|
||||
are expected to be in this page type. A `FixedBlockPage` consists of a number
|
||||
of equally sized blocks, where each allocation will take up exactly one such
|
||||
block.
|
||||
|
||||
```cpp
|
||||
struct FixedBlockCell {
|
||||
union {
|
||||
uint8_t data[];
|
||||
FixedCellRange nextFree;
|
||||
}
|
||||
};
|
||||
|
||||
struct alignas(8) FixedCellRange {
|
||||
uint32_t first;
|
||||
uint32_t last;
|
||||
};
|
||||
```
|
||||
|
||||
Consecutive unallocated cells are represented by a `FixedCellRange`, with
|
||||
`.first` and `.last` being the inclusive end points of the range of unallocated
|
||||
cells. The `FixedBlockCell` at the the `.last` index contains a
|
||||
`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange`
|
||||
of unallocated ranges thus form a linked list.
|
||||
|
||||
The important point is that all links in this list point forward in the page, so
|
||||
all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a
|
||||
`FixedBlockPage` consists of walking the free-list forward, and sweeping all
|
||||
blocks in between the links, maintaining the free list when blocks are freed.
|
||||
|
||||
Each small page takes up the same amount of space, independent of block size,
|
||||
so larger block size implies fewer blocks per page. This size is arbitrarily
|
||||
chosen to be 256 KiB, but this might change.
|
||||
|
||||
## [NextFitPage](cpp/NextFitPage.hpp)
|
||||
|
||||
```cpp
|
||||
class NextFitPage {
|
||||
public:
|
||||
NextFitPage(uint32_t cellCount);
|
||||
Cell* TryAllocate(uint32_t cellCount);
|
||||
bool Sweep();
|
||||
|
||||
private:
|
||||
NextFitPage* next_; // used by AtomicStack
|
||||
Cell* curBlock_;
|
||||
Cell cells_[];
|
||||
};
|
||||
```
|
||||
|
||||
Allocations that could theoretically fit in a `FixedBlockPage`, but would
|
||||
require too large a block size (arbitrary >=1KiB), are allocated in a
|
||||
`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64
|
||||
KiB for experiments). All blocks in a `NextFitPage` have a header that tells how
|
||||
big the block is, and whether it is allocated or not. There are no gaps between
|
||||
blocks, so the size of a block also tells where the next block is. The header
|
||||
information fits inside a 8 byte `Cell`.
|
||||
|
||||
```cpp
|
||||
class Cell {
|
||||
public:
|
||||
Cell(uint32_t size);
|
||||
uint8_t* TryAllocate(uint32_t cellCount);
|
||||
|
||||
private:
|
||||
uint32_t isAllocated_;
|
||||
uint32_t size_;
|
||||
uint8_t data_[];
|
||||
};
|
||||
```
|
||||
|
||||
The page keeps a reference to a currently active block, and will try to bump
|
||||
allocate inside that block. If allocation does not fit, we move to the next
|
||||
block that fits. If no block in the page fits the requested size, the page is
|
||||
abandoned until the next GC.
|
||||
|
||||
## [SingleObjectPage](cpp/SingleObjectPage.hpp)
|
||||
|
||||
```cpp
|
||||
class SingleObjectPage {
|
||||
public:
|
||||
SingleObjectPage(uint64_t cellCount);
|
||||
bool Sweep();
|
||||
|
||||
private:
|
||||
SingleObjectPage* next_; // used by AtomicStack
|
||||
};
|
||||
```
|
||||
|
||||
Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`,
|
||||
which only contains that single block of the requested size. They are also
|
||||
handled slightly differently by both `Heap` and `CustomAllocator`. First off,
|
||||
`Heap::GetSingleObjectPage` will never check existing pages, and instead just
|
||||
allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to
|
||||
any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC
|
||||
thread.
|
||||
|
||||
## [ExtraObjectPage](cpp/ExtraObjectPage.hpp)
|
||||
|
||||
```cpp
|
||||
class ExtraObjectPage {
|
||||
public:
|
||||
ExtraObjectPage();
|
||||
ExtraObjectData* TryAllocate();
|
||||
bool Sweep(FinalizerQueue& queue);
|
||||
|
||||
|
||||
private:
|
||||
ExtraObjectPage* next_; // used by AtomicStack
|
||||
ExtraObjectCell* nextFree_;
|
||||
ExtraObjectCell cells_[];
|
||||
};
|
||||
```
|
||||
|
||||
Extra objects are used for attaching additional data to some objects. This is
|
||||
used for objects that require special handling during garbage collection:
|
||||
|
||||
* objects with finalizers
|
||||
* weak references
|
||||
* interop references
|
||||
|
||||
Extra objects are allocated in `ExtraObjectPage`s, which are very similar to
|
||||
`FixedBlockPage`s. They primarily differ in how they are swept, since it is
|
||||
during sweeping of `ExtraObject`s that scheduling of finalization happens. If
|
||||
an object that requires finalization is found, it is added to the
|
||||
`FinalizerQueue` given as argument. The cells are also slightly different, in
|
||||
that they add a new field that allows the cells to be added to the finalizer
|
||||
queue.
|
||||
|
||||
```cpp
|
||||
struct ExtraObjectCell {
|
||||
ExtraObjectCell* next_; // used by AtomicStack
|
||||
ExtraObjectData data_;
|
||||
};
|
||||
```
|
||||
|
||||
# Finalizers
|
||||
|
||||
Section like to change.
|
||||
|
||||
In the existing memory model, finalization tasks are found and scheduled during
|
||||
sweeping of regular objects. The objects to be finalized are chained together
|
||||
using a pointer in the Node header, added to all allocated objects. This header
|
||||
is not needed in the custom allocator, apart from linking in the finalization
|
||||
queue.
|
||||
|
||||
We therefore reintroduce this pointer in a header for `ExtraObjectData`. For
|
||||
this, we reuse the `ExtraObjectCell` as header for both free list pointer and as
|
||||
linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`.
|
||||
|
||||
# Enabling the allocator
|
||||
|
||||
The custom allocator is enabled with the compiler flag -Xallocator=custom.
|
||||
@@ -0,0 +1,12 @@
|
||||
/*
|
||||
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include "Allocator.hpp"
|
||||
|
||||
using namespace kotlin;
|
||||
|
||||
void alloc::initObjectPool() noexcept {}
|
||||
|
||||
void alloc::compactObjectPoolInCurrentThread() noexcept {}
|
||||
+4
-4
@@ -10,14 +10,14 @@
|
||||
|
||||
using namespace kotlin;
|
||||
|
||||
mm::ExtraObjectData& mm::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject(
|
||||
mm::ExtraObjectData& alloc::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject(
|
||||
ObjHeader* baseObject, const TypeInfo* info) noexcept {
|
||||
return **Emplace(baseObject, info);
|
||||
}
|
||||
|
||||
void mm::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(ExtraObjectData& data) noexcept {
|
||||
void alloc::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept {
|
||||
Erase(&Queue::Node::fromValue(data));
|
||||
}
|
||||
|
||||
mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
|
||||
mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
|
||||
alloc::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
|
||||
alloc::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
|
||||
+5
-10
@@ -3,8 +3,7 @@
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
||||
#define RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
||||
#pragma once
|
||||
|
||||
#include "ExtraObjectData.hpp"
|
||||
#include "Memory.h"
|
||||
@@ -12,8 +11,7 @@
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "ThreadRegistry.hpp"
|
||||
|
||||
namespace kotlin {
|
||||
namespace mm {
|
||||
namespace kotlin::alloc {
|
||||
|
||||
// Registry for extra data, attached to some kotlin objects: weak refs, associated objects, ...
|
||||
class ExtraObjectDataFactory : Pinned {
|
||||
@@ -24,9 +22,9 @@ public:
|
||||
public:
|
||||
explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {}
|
||||
|
||||
ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept;
|
||||
mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept;
|
||||
|
||||
void DestroyExtraObjectData(ExtraObjectData& data) noexcept;
|
||||
void DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept;
|
||||
|
||||
// Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread
|
||||
// when it's asked by GC to stop.
|
||||
@@ -56,7 +54,4 @@ private:
|
||||
Queue extraObjects_;
|
||||
};
|
||||
|
||||
} // namespace mm
|
||||
} // namespace kotlin
|
||||
|
||||
#endif // RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
||||
} // namespace kotlin::alloc
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
#include <cstddef>
|
||||
#include <type_traits>
|
||||
|
||||
namespace kotlin {
|
||||
namespace kotlin::alloc {
|
||||
|
||||
void initObjectPool() noexcept;
|
||||
void* allocateInObjectPool(size_t size) noexcept;
|
||||
+3
-8
@@ -3,8 +3,7 @@
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#ifndef RUNTIME_MM_OBJECT_FACTORY_H
|
||||
#define RUNTIME_MM_OBJECT_FACTORY_H
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
@@ -21,8 +20,7 @@
|
||||
#include "Types.h"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace kotlin {
|
||||
namespace mm {
|
||||
namespace kotlin::alloc {
|
||||
|
||||
namespace internal {
|
||||
|
||||
@@ -736,7 +734,4 @@ private:
|
||||
Storage storage_;
|
||||
};
|
||||
|
||||
} // namespace mm
|
||||
} // namespace kotlin
|
||||
|
||||
#endif // RUNTIME_MM_OBJECT_FACTORY_H
|
||||
} // namespace kotlin::alloc
|
||||
+3
-5
@@ -9,14 +9,13 @@
|
||||
|
||||
#include "ObjectAlloc.hpp"
|
||||
|
||||
namespace kotlin {
|
||||
namespace gc {
|
||||
namespace kotlin::alloc {
|
||||
|
||||
// TODO: Try to move from custom allocator interface to standard one.
|
||||
// Currently Free method is in the way: it is static to avoid keeping allocator state in
|
||||
// unique_ptr's deleter in ObjectFactory.
|
||||
|
||||
class Allocator {
|
||||
class AllocatorBasic {
|
||||
public:
|
||||
void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); }
|
||||
static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); }
|
||||
@@ -43,5 +42,4 @@ private:
|
||||
GCThreadData& gc_;
|
||||
};
|
||||
|
||||
} // namespace gc
|
||||
} // namespace kotlin
|
||||
} // namespace kotlin::alloc
|
||||
+4
-4
@@ -3,7 +3,7 @@
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include "Allocator.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
|
||||
#include "gmock/gmock.h"
|
||||
#include "gtest/gtest.h"
|
||||
@@ -48,7 +48,7 @@ TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
|
||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
||||
EXPECT_CALL(gc, OnOOM(_)).Times(0);
|
||||
}
|
||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
void* ptr = allocator.Alloc(size);
|
||||
EXPECT_THAT(ptr, nonNull);
|
||||
}
|
||||
@@ -64,7 +64,7 @@ TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
|
||||
EXPECT_CALL(gc, OnOOM(size));
|
||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
||||
}
|
||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
void* ptr = allocator.Alloc(size);
|
||||
EXPECT_THAT(ptr, nonNull);
|
||||
}
|
||||
@@ -79,7 +79,7 @@ TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
|
||||
EXPECT_CALL(gc, OnOOM(size));
|
||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
|
||||
}
|
||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||
void* ptr = allocator.Alloc(size);
|
||||
EXPECT_THAT(ptr, nullptr);
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "FinalizerHooks.hpp"
|
||||
#include "GC.hpp"
|
||||
#include "GCStatistics.hpp"
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
|
||||
namespace kotlin::alloc {
|
||||
|
||||
template <typename Traits>
|
||||
void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept {
|
||||
auto sweepHandle = handle.sweepExtraObjects();
|
||||
factoryIter.ApplyDeletions();
|
||||
for (auto it = factoryIter.begin(); it != factoryIter.end();) {
|
||||
auto &extraObject = *it;
|
||||
if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) {
|
||||
extraObject.ClearRegularWeakReferenceImpl();
|
||||
if (extraObject.HasAssociatedObject()) {
|
||||
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
} else {
|
||||
extraObject.Uninstall();
|
||||
it.EraseAndAdvance();
|
||||
sweepHandle.addSweptObject();
|
||||
}
|
||||
} else {
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept {
|
||||
auto iter = factory.LockForIter();
|
||||
return SweepExtraObjects<Traits>(handle, iter);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
|
||||
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
|
||||
auto sweepHandle = handle.sweep();
|
||||
|
||||
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
|
||||
if (Traits::TryResetMark(*it)) {
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
continue;
|
||||
}
|
||||
sweepHandle.addSweptObject();
|
||||
auto* objHeader = it->GetObjHeader();
|
||||
if (HasFinalizers(objHeader)) {
|
||||
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
|
||||
} else {
|
||||
objectFactoryIter.EraseAndAdvance(it);
|
||||
}
|
||||
}
|
||||
|
||||
return finalizerQueue;
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
|
||||
auto iter = objectFactory.LockForIter();
|
||||
return Sweep<Traits>(handle, iter);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct DefaultSweepTraits {
|
||||
using ObjectFactory = T;
|
||||
using ExtraObjectsFactory = ExtraObjectDataFactory;
|
||||
|
||||
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
|
||||
auto* baseObject = object.GetBaseObject();
|
||||
if (!baseObject->heap()) return true;
|
||||
return gc::isMarked(baseObject);
|
||||
}
|
||||
|
||||
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
|
||||
};
|
||||
|
||||
}
|
||||
+14
-13
@@ -3,17 +3,18 @@
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include <TestSupport.hpp>
|
||||
#include "MarkAndSweepUtils.hpp"
|
||||
#include "ObjectFactorySweep.hpp"
|
||||
|
||||
#include "gmock/gmock.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
#include "Allocator.hpp"
|
||||
#include "FinalizerHooksTestSupport.hpp"
|
||||
#include "ObjectFactory.hpp"
|
||||
#include "ObjectTestSupport.hpp"
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
#include "FinalizerHooksTestSupport.hpp"
|
||||
#include "MarkAndSweepUtils.hpp"
|
||||
#include "ObjectFactory.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
#include "ObjectTestSupport.hpp"
|
||||
#include "TestSupport.hpp"
|
||||
#include "WeakRef.hpp"
|
||||
|
||||
using namespace kotlin;
|
||||
@@ -45,11 +46,11 @@ struct ObjectFactoryTraits {
|
||||
State state = State::kUnmarked;
|
||||
};
|
||||
|
||||
using Allocator = gc::Allocator;
|
||||
using Allocator = alloc::AllocatorBasic;
|
||||
};
|
||||
|
||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ExtraObjectsDataFactory = mm::ExtraObjectDataFactory;
|
||||
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ExtraObjectsDataFactory = alloc::ExtraObjectDataFactory;
|
||||
|
||||
class Object : public test_support::Object<Payload> {
|
||||
public:
|
||||
@@ -134,7 +135,7 @@ ObjectFactoryTraits::ObjectData::State GetWeakReferenceState(test_support::Regul
|
||||
|
||||
struct SweepTraits {
|
||||
using ObjectFactory = ObjectFactory;
|
||||
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
|
||||
using ExtraObjectsFactory = alloc::ExtraObjectDataFactory;
|
||||
|
||||
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
|
||||
auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).ObjectData();
|
||||
@@ -195,8 +196,8 @@ public:
|
||||
|
||||
std_support::vector<ObjHeader*> Sweep() {
|
||||
gc::processWeaks<ProcessWeakTraits>(gc::GCHandle::getByEpoch(0), specialRefRegistry_);
|
||||
gc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
|
||||
auto finalizers = gc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
|
||||
alloc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
|
||||
auto finalizers = alloc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
|
||||
std_support::vector<ObjHeader*> objects;
|
||||
for (auto node : finalizers.IterForTests()) {
|
||||
objects.push_back(node.GetObjHeader());
|
||||
@@ -266,7 +267,7 @@ private:
|
||||
kotlin::ScopedMemoryInit memoryInit;
|
||||
FinalizerHooksTestSupport finalizerHooks_;
|
||||
ObjectFactory objectFactory_;
|
||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gc::Allocator()};
|
||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, alloc::AllocatorBasic()};
|
||||
ExtraObjectsDataFactory extraObjectFactory_;
|
||||
ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_};
|
||||
mm::SpecialRefRegistry specialRefRegistry_;
|
||||
+3
-3
@@ -11,8 +11,8 @@
|
||||
#include "gmock/gmock.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
#include "Allocator.hpp"
|
||||
#include "FinalizerHooksTestSupport.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
#include "ObjectOps.hpp"
|
||||
#include "ObjectTestSupport.hpp"
|
||||
#include "ScopedThread.hpp"
|
||||
@@ -39,7 +39,7 @@ struct DataSizeProvider {
|
||||
};
|
||||
|
||||
template <size_t DataAlignment>
|
||||
using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator, DataSizeProvider>;
|
||||
using ObjectFactoryStorage = alloc::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator, DataSizeProvider>;
|
||||
|
||||
using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
|
||||
|
||||
@@ -848,7 +848,7 @@ public:
|
||||
using Allocator = GlobalMockAllocator;
|
||||
};
|
||||
|
||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||
|
||||
struct Payload {
|
||||
ObjHeader* field1;
|
||||
+8
-8
@@ -7,10 +7,10 @@
|
||||
|
||||
#include <mutex>
|
||||
|
||||
#include "../../mimalloc/c/include/mimalloc.h"
|
||||
#include "Alignment.hpp"
|
||||
#include "CompilerConstants.hpp"
|
||||
#include "Memory.h"
|
||||
#include "mimalloc.h"
|
||||
|
||||
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
||||
#include <dispatch/dispatch.h>
|
||||
@@ -28,7 +28,7 @@ std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT;
|
||||
|
||||
} // namespace
|
||||
|
||||
void kotlin::initObjectPool() noexcept {
|
||||
void alloc::initObjectPool() noexcept {
|
||||
if (!compiler::mimallocUseDefaultOptions()) {
|
||||
std::call_once(initOptions, [] {
|
||||
mi_option_enable(mi_option_reset_decommits);
|
||||
@@ -40,20 +40,20 @@ void kotlin::initObjectPool() noexcept {
|
||||
mi_thread_init();
|
||||
}
|
||||
|
||||
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
||||
void* alloc::allocateInObjectPool(size_t size) noexcept {
|
||||
return mi_calloc_aligned(1, size, kObjectAlignment);
|
||||
}
|
||||
|
||||
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
void alloc::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
mi_free(ptr);
|
||||
}
|
||||
|
||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {
|
||||
void alloc::compactObjectPoolInCurrentThread() noexcept {
|
||||
if (!compiler::mimallocUseCompaction()) return;
|
||||
mi_collect(true);
|
||||
}
|
||||
|
||||
void kotlin::compactObjectPoolInMainThread() noexcept {
|
||||
void alloc::compactObjectPoolInMainThread() noexcept {
|
||||
if (!compiler::mimallocUseCompaction()) return;
|
||||
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
||||
if (scheduledCompactOnMainThread.test_and_set()) {
|
||||
@@ -62,14 +62,14 @@ void kotlin::compactObjectPoolInMainThread() noexcept {
|
||||
}
|
||||
dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) {
|
||||
if (mm::IsCurrentThreadRegistered()) {
|
||||
kotlin::compactObjectPoolInCurrentThread();
|
||||
alloc::compactObjectPoolInCurrentThread();
|
||||
}
|
||||
scheduledCompactOnMainThread.clear();
|
||||
});
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t kotlin::allocatedBytes() noexcept {
|
||||
size_t alloc::allocatedBytes() noexcept {
|
||||
return mi_allocated_size();
|
||||
}
|
||||
|
||||
+6
-6
@@ -36,9 +36,9 @@ size_t allocatedBytesCounter = 0;
|
||||
|
||||
} // namespace
|
||||
|
||||
void kotlin::initObjectPool() noexcept {}
|
||||
void alloc::initObjectPool() noexcept {}
|
||||
|
||||
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
||||
void* alloc::allocateInObjectPool(size_t size) noexcept {
|
||||
// TODO: Check that alignment to kObjectAlignment is satisfied.
|
||||
void* result = callocImpl(1, size);
|
||||
#ifndef KONAN_NO_THREADS
|
||||
@@ -52,7 +52,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
||||
return result;
|
||||
}
|
||||
|
||||
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
void alloc::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
#ifndef KONAN_NO_THREADS
|
||||
allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed);
|
||||
#else
|
||||
@@ -61,11 +61,11 @@ void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
freeImpl(ptr);
|
||||
}
|
||||
|
||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {}
|
||||
void alloc::compactObjectPoolInCurrentThread() noexcept {}
|
||||
|
||||
void kotlin::compactObjectPoolInMainThread() noexcept {}
|
||||
void alloc::compactObjectPoolInMainThread() noexcept {}
|
||||
|
||||
size_t kotlin::allocatedBytes() noexcept {
|
||||
size_t alloc::allocatedBytes() noexcept {
|
||||
#ifndef KONAN_NO_THREADS
|
||||
return allocatedBytesCounter.load();
|
||||
#else
|
||||
@@ -1,372 +1 @@
|
||||
# Overview of the allocator
|
||||
|
||||
This document describes the internals of the custom allocator. The presentation
|
||||
here is not fully true to the implementation, as the design is still work in
|
||||
progress.
|
||||
|
||||
The main idea of the custom allocator is to divide system memory into chunks
|
||||
(pages) that each can be swept independently, in memory consecutive order.
|
||||
Every allocation ends up as a block of memory inside a page. Each page keeps
|
||||
track of the size of each block it contains; how this is done depends on the
|
||||
page type, with different page types optimized for different allocation sizes.
|
||||
All the memory blocks are consecutive within a page, so the size of a block
|
||||
also tells where the next block begins. Paired with an additional mechanism
|
||||
(page type dependent) for determining whether a block is allocated, we can
|
||||
iterate through the allocated blocks.
|
||||
|
||||
When a thread allocates memory for an object, it will find a page suitable for
|
||||
the given allocation size. Each thread holds on to a number of pages for the
|
||||
different size categories. The typical case is that the thread’s current page
|
||||
for the given size can fit the requested allocation. If that is not the case, a
|
||||
different page for that size category is requested from the shared allocation
|
||||
space. The requested page can either be readily available (already prepared by
|
||||
the GC thread), or it might need to be swept first, or it might be newly
|
||||
created.
|
||||
|
||||
The GC thread has a new responsibility when using this allocator: While the
|
||||
mutator threads are paused at the start of GC, the GC thread must prepare the
|
||||
allocator for sweeping. This does two things. First, it marks all pages as
|
||||
“needs to be swept before next use”. Second, it releases pages that threads are
|
||||
holding on to, by clearing the thread local variables for each thread.
|
||||
|
||||
It is possible to have several independent allocation spaces at the same time.
|
||||
This is currently useful for testing, but is potentially useful in other
|
||||
settings.
|
||||
|
||||
# Detailed Design
|
||||
|
||||
All allocations are made through a `CustomAllocator` object.
|
||||
|
||||
## [CustomAllocator](cpp/CustomAllocator.hpp)
|
||||
|
||||
```cpp
|
||||
class CustomAllocator {
|
||||
public:
|
||||
CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler);
|
||||
ObjectHeader* CreateObject(TypeInfo* type);
|
||||
ArrayHeader* CreateArray(TypeInfo* type, uint32_t count);
|
||||
ExtraObjectData* CreateExtraObject();
|
||||
void PrepareForGc();
|
||||
|
||||
private:
|
||||
uint8_t* Allocate(uint64_t cellCount);
|
||||
uint8_t* AllocateInSingleObjectPage(uint64_t cellCount);
|
||||
uint8_t* AllocateInNextFitPage(uint32_t cellCount);
|
||||
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount);
|
||||
|
||||
Heap& heap_;
|
||||
GCSchedulerThreadData& gcScheduler_;
|
||||
NextFitPage* nextFitPage_;
|
||||
FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||
ExtraObjectPage* extraObjectPage_;
|
||||
};
|
||||
```
|
||||
|
||||
The primary responsibility of this class is to delegate each requested
|
||||
allocation to pages of the appropriate type, based on allocation size. To do
|
||||
this, it requests pages from the shared allocation space (`Heap`) and stores
|
||||
pages for later allocations. Each thread thus owns a number of pages for
|
||||
different allocation sizes, but at most one for each size class. When
|
||||
allocating, the `CustomAllocator` will first try to allocate in one of its
|
||||
owned pages. If this fails, it will request a new page for that size class from
|
||||
a shared `Heap` object. `SingleObjectPages` are never kept by the
|
||||
`CustomAllocator`, since they are created specifically for a single allocation,
|
||||
with no extra space.
|
||||
|
||||
## [Heap](cpp/Heap.hpp)
|
||||
|
||||
```cpp
|
||||
class Heap {
|
||||
public:
|
||||
void PrepareForGC();
|
||||
|
||||
void Sweep();
|
||||
|
||||
AtomicStack<ExtraObjectCell> SweepExtraObjects(GCHandle gcHandle);
|
||||
|
||||
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount);
|
||||
NextFitPage* GetNextFitPage(uint32_t cellCount);
|
||||
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount);
|
||||
ExtraObjectPage* GetExtraObjectPage();
|
||||
|
||||
private:
|
||||
PageStore<FixedBlockPage> fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||
PageStore<NextFitPage> nextFitPages_;
|
||||
PageStore<SingleObjectPage> singleObjectPages_;
|
||||
AtomicStack<ExtraObjectPage> extraObjectPages_;
|
||||
AtomicStack<ExtraObjectPage> usedExtraObjectPages_;
|
||||
};
|
||||
```
|
||||
|
||||
A `Heap` object represents a shared allocation space for multiple
|
||||
`CustomAllocator`s, which can request pages through one of the
|
||||
`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also
|
||||
provides a method for sweeping through all blocks that have been allocated in
|
||||
this heap. The `Heap` object is the synchronization point, and guarantees that
|
||||
every page is returned at most once. Page ownership is thus implicitly given to
|
||||
the thread that called the method. The `Heap` object keeps track of all pages,
|
||||
so there is no need to explicitly return ownership of a page. Internally, a
|
||||
`Heap` keeps the pages for each size class in a `PageStore`. This means one for
|
||||
`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes
|
||||
for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two
|
||||
`AtomicStack`s, since they require different handling during sweeping.
|
||||
|
||||
## [PageStore](cpp/PageStore.hpp)
|
||||
|
||||
```cpp
|
||||
template <class PageType>
|
||||
class PageStore {
|
||||
public:
|
||||
void PrepareForGC();
|
||||
void Sweep();
|
||||
void SweepAndFree();
|
||||
PageType* GetPage(uint32_t cellCount);
|
||||
PageType* NewPage(uint64_t cellCount);
|
||||
|
||||
private:
|
||||
AtomicStack<PageType> empty_;
|
||||
AtomicStack<PageType> ready_;
|
||||
AtomicStack<PageType> used_;
|
||||
AtomicStack<PageType> unswept_;
|
||||
};
|
||||
```
|
||||
|
||||
A PageStore is responsible for keeping track of all pages of a given type and
|
||||
size class. Each of the pages are in one of four stacks. The stack, that a
|
||||
given page is in, determines its current state:
|
||||
|
||||
* `unswept_`: have not yet been swept since the last GC cycle.
|
||||
* `ready_`: are ready for allocation; has been swept by the GC thread.
|
||||
* `used_`: has been given to some thread for allocation; it might still be used
|
||||
for allocation, or it might have been discarded with not enough space left.
|
||||
Will not be used until the next GC cycle.
|
||||
* `empty_`: same as `ready_`, but does not contain any objects. Will be freed
|
||||
before the next GC, if not needed before then.
|
||||
|
||||
When a page is requested, the page is taken from `ready_`, if there are any.
|
||||
Otherwise, an `unswept_` page is taken and swept before returning. If there are
|
||||
no unswept pages either, an empty page is taken, if there are any. Otherwise a
|
||||
new page is created in the size category. All returned pages are moved to
|
||||
`used_`. During the marking phase, all remaining pages in `empty_` are freed,
|
||||
and all other pages are moved to `unswept_`. The GC thread will go through all
|
||||
`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one
|
||||
of the other threads sweeps a page from `unswept_`, it is moved directly to
|
||||
`used_`, as it is claimed by the `CustomAllocator` that swept it.
|
||||
|
||||
`SingleObjectPage`s are treated slightly differently, because they are created
|
||||
for one specific single allocation, and not reused when that allocation is
|
||||
freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without
|
||||
checking any of the stacks, and during sweeping, they are freed directly rather
|
||||
than being put into the `empty_` stack. Ideally, there would only be two stacks
|
||||
in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little
|
||||
is lost by just using the existing `PageStore` logic used for the other pages.
|
||||
|
||||
## [AtomicStack](cpp/AtomicStack.hpp)
|
||||
|
||||
```cpp
|
||||
template <class PageType>
|
||||
class AtomicStack {
|
||||
public:
|
||||
PageType* Pop();
|
||||
void Push(PageType* elm);
|
||||
void TransferAllFrom(AtomicStack<T>& src);
|
||||
bool isEmpty();
|
||||
|
||||
|
||||
private:
|
||||
std::atomic<PageType*> stack_;
|
||||
};
|
||||
```
|
||||
|
||||
The only place where atomics are used are in the stacks inside the `PageStore`.
|
||||
All page classes have a non-atomic next pointer, to be used for linking up in
|
||||
exactly one stack. `Pop` and `Push` are implemented with compare-and-swap
|
||||
operations. The class is thread safe, except for if an element is freed while
|
||||
another thread tries to Pop it from a stack.
|
||||
|
||||
# Page types
|
||||
|
||||
This section is likely to change, given the likely introduction of additional
|
||||
page types. It also describes some details about which page type is chosen for
|
||||
a given allocation, which is also likely to change.
|
||||
|
||||
There are four different page types, but they all share the feature that they
|
||||
can be swept independently. The Sweep methods return whether there were any
|
||||
live objects in the page after sweeping. If not, the page will be given back to
|
||||
the OS.
|
||||
|
||||
## [FixedBlockPage](cpp/FixedBlockPage.hpp)
|
||||
|
||||
```cpp
|
||||
class FixedBlockPage {
|
||||
public:
|
||||
FixedBlockPage(uint32_t blockSize) noexcept;
|
||||
|
||||
uint8_t* TryAllocate() noexcept;
|
||||
|
||||
bool Sweep() noexcept;
|
||||
|
||||
private:
|
||||
FixedBlockPage* next_;
|
||||
FixedCellRange nextFree_;
|
||||
uint32_t blockSize_;
|
||||
uint32_t end_;
|
||||
FixedBlockCell cells_[];
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
All sufficiently small allocations (currently arbitrary <1KiB) are directed to
|
||||
a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations
|
||||
are expected to be in this page type. A `FixedBlockPage` consists of a number
|
||||
of equally sized blocks, where each allocation will take up exactly one such
|
||||
block.
|
||||
|
||||
```cpp
|
||||
struct FixedBlockCell {
|
||||
union {
|
||||
uint8_t data[];
|
||||
FixedCellRange nextFree;
|
||||
}
|
||||
};
|
||||
|
||||
struct alignas(8) FixedCellRange {
|
||||
uint32_t first;
|
||||
uint32_t last;
|
||||
};
|
||||
```
|
||||
|
||||
Consecutive unallocated cells are represented by a `FixedCellRange`, with
|
||||
`.first` and `.last` being the inclusive end points of the range of unallocated
|
||||
cells. The `FixedBlockCell` at the the `.last` index contains a
|
||||
`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange`
|
||||
of unallocated ranges thus form a linked list.
|
||||
|
||||
The important point is that all links in this list point forward in the page, so
|
||||
all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a
|
||||
`FixedBlockPage` consists of walking the free-list forward, and sweeping all
|
||||
blocks in between the links, maintaining the free list when blocks are freed.
|
||||
|
||||
Each small page takes up the same amount of space, independent of block size,
|
||||
so larger block size implies fewer blocks per page. This size is arbitrarily
|
||||
chosen to be 256 KiB, but this might change.
|
||||
|
||||
## [NextFitPage](cpp/NextFitPage.hpp)
|
||||
|
||||
```cpp
|
||||
class NextFitPage {
|
||||
public:
|
||||
NextFitPage(uint32_t cellCount);
|
||||
Cell* TryAllocate(uint32_t cellCount);
|
||||
bool Sweep();
|
||||
|
||||
private:
|
||||
NextFitPage* next_; // used by AtomicStack
|
||||
Cell* curBlock_;
|
||||
Cell cells_[];
|
||||
};
|
||||
```
|
||||
|
||||
Allocations that could theoretically fit in a `FixedBlockPage`, but would
|
||||
require too large a block size (arbitrary >=1KiB), are allocated in a
|
||||
`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64
|
||||
KiB for experiments). All blocks in a `NextFitPage` have a header that tells how
|
||||
big the block is, and whether it is allocated or not. There are no gaps between
|
||||
blocks, so the size of a block also tells where the next block is. The header
|
||||
information fits inside a 8 byte `Cell`.
|
||||
|
||||
```cpp
|
||||
class Cell {
|
||||
public:
|
||||
Cell(uint32_t size);
|
||||
uint8_t* TryAllocate(uint32_t cellCount);
|
||||
|
||||
private:
|
||||
uint32_t isAllocated_;
|
||||
uint32_t size_;
|
||||
uint8_t data_[];
|
||||
};
|
||||
```
|
||||
|
||||
The page keeps a reference to a currently active block, and will try to bump
|
||||
allocate inside that block. If allocation does not fit, we move to the next
|
||||
block that fits. If no block in the page fits the requested size, the page is
|
||||
abandoned until the next GC.
|
||||
|
||||
## [SingleObjectPage](cpp/SingleObjectPage.hpp)
|
||||
|
||||
```cpp
|
||||
class SingleObjectPage {
|
||||
public:
|
||||
SingleObjectPage(uint64_t cellCount);
|
||||
bool Sweep();
|
||||
|
||||
private:
|
||||
SingleObjectPage* next_; // used by AtomicStack
|
||||
};
|
||||
```
|
||||
|
||||
Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`,
|
||||
which only contains that single block of the requested size. They are also
|
||||
handled slightly differently by both `Heap` and `CustomAllocator`. First off,
|
||||
`Heap::GetSingleObjectPage` will never check existing pages, and instead just
|
||||
allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to
|
||||
any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC
|
||||
thread.
|
||||
|
||||
## [ExtraObjectPage](cpp/ExtraObjectPage.hpp)
|
||||
|
||||
```cpp
|
||||
class ExtraObjectPage {
|
||||
public:
|
||||
ExtraObjectPage();
|
||||
ExtraObjectData* TryAllocate();
|
||||
bool Sweep(FinalizerQueue& queue);
|
||||
|
||||
|
||||
private:
|
||||
ExtraObjectPage* next_; // used by AtomicStack
|
||||
ExtraObjectCell* nextFree_;
|
||||
ExtraObjectCell cells_[];
|
||||
};
|
||||
```
|
||||
|
||||
Extra objects are used for attaching additional data to some objects. This is
|
||||
used for objects that require special handling during garbage collection:
|
||||
|
||||
* objects with finalizers
|
||||
* weak references
|
||||
* interop references
|
||||
|
||||
Extra objects are allocated in `ExtraObjectPage`s, which are very similar to
|
||||
`FixedBlockPage`s. They primarily differ in how they are swept, since it is
|
||||
during sweeping of `ExtraObject`s that scheduling of finalization happens. If
|
||||
an object that requires finalization is found, it is added to the
|
||||
`FinalizerQueue` given as argument. The cells are also slightly different, in
|
||||
that they add a new field that allows the cells to be added to the finalizer
|
||||
queue.
|
||||
|
||||
```cpp
|
||||
struct ExtraObjectCell {
|
||||
ExtraObjectCell* next_; // used by AtomicStack
|
||||
ExtraObjectData data_;
|
||||
};
|
||||
```
|
||||
|
||||
# Finalizers
|
||||
|
||||
Section like to change.
|
||||
|
||||
In the existing memory model, finalization tasks are found and scheduled during
|
||||
sweeping of regular objects. The objects to be finalized are chained together
|
||||
using a pointer in the Node header, added to all allocated objects. This header
|
||||
is not needed in the custom allocator, apart from linking in the finalization
|
||||
queue.
|
||||
|
||||
We therefore reintroduce this pointer in a header for `ExtraObjectData`. For
|
||||
this, we reuse the `ExtraObjectCell` as header for both free list pointer and as
|
||||
linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`.
|
||||
|
||||
# Enabling the allocator
|
||||
|
||||
The custom allocator is enabled with the compiler flag -Xallocator=custom.
|
||||
Allocator documentation was moved [here](../alloc/custom/README.md).
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include "CustomLogging.hpp"
|
||||
|
||||
#include "GCApi.hpp"
|
||||
|
||||
// These functions are just stubs to make the existing object creation
|
||||
// infrastructure link correctly, but if they are ever called, something went
|
||||
// wrong.
|
||||
|
||||
namespace kotlin {
|
||||
|
||||
void* allocateInObjectPool(size_t size) noexcept {
|
||||
CustomAllocWarning("static allocateInObjectPool(%zu) not supported", size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||
CustomAllocWarning("static freeInObjectPool(%p, %zu) not supported", ptr, size);
|
||||
}
|
||||
|
||||
void initObjectPool() noexcept {}
|
||||
void compactObjectPoolInMainThread() noexcept {}
|
||||
void compactObjectPoolInCurrentThread() noexcept {}
|
||||
|
||||
size_t allocatedBytes() noexcept {
|
||||
return alloc::GetAllocatedBytes();
|
||||
}
|
||||
|
||||
} // namespace kotlin
|
||||
@@ -7,6 +7,8 @@
|
||||
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
|
||||
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||
|
||||
#include "CustomAllocator.hpp"
|
||||
#include "CustomFinalizerProcessor.hpp"
|
||||
#include "GCApi.hpp"
|
||||
@@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
|
||||
|
||||
#else
|
||||
|
||||
#include "Allocator.hpp"
|
||||
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
#include "GC.hpp"
|
||||
#include "GlobalData.hpp"
|
||||
#include "Logging.hpp"
|
||||
#include "ObjectFactory.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
#include "ObjectFactorySweep.hpp"
|
||||
|
||||
namespace kotlin::gc {
|
||||
|
||||
struct ObjectFactoryTraits {
|
||||
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
|
||||
using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
|
||||
using ObjectData = gc::GC::ObjectData;
|
||||
|
||||
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
|
||||
Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
|
||||
|
||||
void OnOOM(size_t size) noexcept {
|
||||
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
||||
@@ -51,7 +56,7 @@ struct ObjectFactoryTraits {
|
||||
}
|
||||
};
|
||||
|
||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||
|
||||
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
||||
return ObjectFactory::NodeRef::From(object).ObjectData();
|
||||
|
||||
@@ -92,7 +92,7 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
|
||||
ObjectFactory& objectFactory,
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
gcScheduler::GCScheduler& gcScheduler,
|
||||
bool mutatorsCooperate,
|
||||
std::size_t auxGCThreads) noexcept :
|
||||
@@ -211,11 +211,11 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
||||
resumeTheWorld(gcHandle);
|
||||
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||
extraObjectFactoryIterable = std::nullopt;
|
||||
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||
auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||
objectFactoryIterable = std::nullopt;
|
||||
kotlin::compactObjectPoolInMainThread();
|
||||
alloc::compactObjectPoolInMainThread();
|
||||
#else
|
||||
// also sweeps extraObjects
|
||||
auto finalizerQueue = heap_.Sweep(gcHandle);
|
||||
@@ -224,7 +224,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
||||
}
|
||||
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
||||
#endif
|
||||
scheduler.onGCFinish(epoch, allocatedBytes());
|
||||
scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
|
||||
state_.finish(epoch);
|
||||
gcHandle.finalizersScheduled(finalizerQueue.size());
|
||||
gcHandle.finished();
|
||||
|
||||
@@ -67,7 +67,7 @@ public:
|
||||
#else
|
||||
ConcurrentMarkAndSweep(
|
||||
ObjectFactory& objectFactory,
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
gcScheduler::GCScheduler& scheduler,
|
||||
bool mutatorsCooperate,
|
||||
std::size_t auxGCThreads) noexcept;
|
||||
@@ -93,7 +93,7 @@ private:
|
||||
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory& objectFactory_;
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||
#else
|
||||
alloc::Heap heap_;
|
||||
#endif
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
#include "GC.hpp"
|
||||
#include "GCStatistics.hpp"
|
||||
#include "MarkAndSweepUtils.hpp"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "ObjectOps.hpp"
|
||||
#include "std_support/Memory.hpp"
|
||||
|
||||
@@ -100,7 +99,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
||||
}
|
||||
|
||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||
return allocatedBytes();
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
return alloc::GetAllocatedBytes();
|
||||
#else
|
||||
return alloc::allocatedBytes();
|
||||
#endif
|
||||
}
|
||||
|
||||
void gc::GC::ClearForTests() noexcept {
|
||||
|
||||
@@ -23,14 +23,14 @@ public:
|
||||
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
#endif
|
||||
ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
|
||||
|
||||
private:
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory objectFactory_;
|
||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
#endif
|
||||
ConcurrentMarkAndSweep gc_;
|
||||
};
|
||||
@@ -53,7 +53,7 @@ public:
|
||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||
#else
|
||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
#endif
|
||||
|
||||
private:
|
||||
@@ -63,7 +63,7 @@ private:
|
||||
#else
|
||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
|
||||
|
||||
#include "ExtraObjectData.hpp"
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
#include "FinalizerHooks.hpp"
|
||||
#include "GlobalData.hpp"
|
||||
#include "GCStatistics.hpp"
|
||||
@@ -106,79 +105,6 @@ void Mark(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQue
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept {
|
||||
auto sweepHandle = handle.sweepExtraObjects();
|
||||
factoryIter.ApplyDeletions();
|
||||
for (auto it = factoryIter.begin(); it != factoryIter.end();) {
|
||||
auto &extraObject = *it;
|
||||
if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) {
|
||||
extraObject.ClearRegularWeakReferenceImpl();
|
||||
if (extraObject.HasAssociatedObject()) {
|
||||
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
} else {
|
||||
extraObject.Uninstall();
|
||||
it.EraseAndAdvance();
|
||||
sweepHandle.addSweptObject();
|
||||
}
|
||||
} else {
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept {
|
||||
auto iter = factory.LockForIter();
|
||||
return SweepExtraObjects<Traits>(handle, iter);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
|
||||
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
|
||||
auto sweepHandle = handle.sweep();
|
||||
|
||||
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
|
||||
if (Traits::TryResetMark(*it)) {
|
||||
++it;
|
||||
sweepHandle.addKeptObject();
|
||||
continue;
|
||||
}
|
||||
sweepHandle.addSweptObject();
|
||||
auto* objHeader = it->GetObjHeader();
|
||||
if (HasFinalizers(objHeader)) {
|
||||
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
|
||||
} else {
|
||||
objectFactoryIter.EraseAndAdvance(it);
|
||||
}
|
||||
}
|
||||
|
||||
return finalizerQueue;
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
|
||||
auto iter = objectFactory.LockForIter();
|
||||
return Sweep<Traits>(handle, iter);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct DefaultSweepTraits {
|
||||
using ObjectFactory = T;
|
||||
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
|
||||
|
||||
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
|
||||
auto* baseObject = object.GetBaseObject();
|
||||
if (!baseObject->heap()) return true;
|
||||
return gc::isMarked(baseObject);
|
||||
}
|
||||
|
||||
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
|
||||
};
|
||||
|
||||
template <typename Traits>
|
||||
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
|
||||
auto handle = gcHandle.collectThreadRoots(thread);
|
||||
|
||||
@@ -7,26 +7,31 @@
|
||||
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
|
||||
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||
|
||||
#include "CustomAllocator.hpp"
|
||||
#include "GCApi.hpp"
|
||||
#include "Heap.hpp"
|
||||
|
||||
#else
|
||||
|
||||
#include "Allocator.hpp"
|
||||
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
#include "GC.hpp"
|
||||
#include "ObjectFactory.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
|
||||
namespace kotlin::gc {
|
||||
|
||||
struct ObjectFactoryTraits {
|
||||
using Allocator = Allocator;
|
||||
using Allocator = alloc::AllocatorBasic;
|
||||
using ObjectData = gc::GC::ObjectData;
|
||||
|
||||
Allocator CreateAllocator() noexcept { return Allocator(); }
|
||||
};
|
||||
|
||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||
|
||||
} // namespace kotlin::gc
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
#include "GC.hpp"
|
||||
#include "GCStatistics.hpp"
|
||||
#include "NoOpGC.hpp"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "ObjectOps.hpp"
|
||||
#include "ThreadData.hpp"
|
||||
#include "std_support/Memory.hpp"
|
||||
@@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
||||
}
|
||||
|
||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||
return allocatedBytes();
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
return alloc::GetAllocatedBytes();
|
||||
#else
|
||||
return alloc::allocatedBytes();
|
||||
#endif
|
||||
}
|
||||
|
||||
void gc::GC::ClearForTests() noexcept {
|
||||
|
||||
@@ -19,14 +19,14 @@ public:
|
||||
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
#endif
|
||||
NoOpGC& gc() noexcept { return gc_; }
|
||||
|
||||
private:
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory objectFactory_;
|
||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
#endif
|
||||
NoOpGC gc_;
|
||||
};
|
||||
@@ -46,7 +46,7 @@ public:
|
||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||
#else
|
||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
#endif
|
||||
|
||||
private:
|
||||
@@ -56,7 +56,7 @@ private:
|
||||
#else
|
||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
|
||||
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||
|
||||
#include "CustomAllocator.hpp"
|
||||
#include "CustomFinalizerProcessor.hpp"
|
||||
#include "GCApi.hpp"
|
||||
@@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
|
||||
|
||||
#else
|
||||
|
||||
#include "Allocator.hpp"
|
||||
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||
|
||||
#include "ExtraObjectDataFactory.hpp"
|
||||
#include "GC.hpp"
|
||||
#include "GlobalData.hpp"
|
||||
#include "Logging.hpp"
|
||||
#include "ObjectFactory.hpp"
|
||||
#include "ObjectFactoryAllocator.hpp"
|
||||
#include "ObjectFactorySweep.hpp"
|
||||
|
||||
namespace kotlin::gc {
|
||||
|
||||
struct ObjectFactoryTraits {
|
||||
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
|
||||
using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
|
||||
using ObjectData = gc::GC::ObjectData;
|
||||
|
||||
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
|
||||
Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
|
||||
|
||||
void OnOOM(size_t size) noexcept {
|
||||
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
||||
@@ -51,7 +56,7 @@ struct ObjectFactoryTraits {
|
||||
}
|
||||
};
|
||||
|
||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
||||
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||
|
||||
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
||||
return ObjectFactory::NodeRef::From(object).ObjectData();
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
#include "GCStatistics.hpp"
|
||||
#include "GlobalData.hpp"
|
||||
#include "MarkAndSweepUtils.hpp"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "ObjectOps.hpp"
|
||||
#include "SameThreadMarkAndSweep.hpp"
|
||||
#include "std_support/Memory.hpp"
|
||||
@@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
||||
}
|
||||
|
||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||
return allocatedBytes();
|
||||
#ifdef CUSTOM_ALLOCATOR
|
||||
return alloc::GetAllocatedBytes();
|
||||
#else
|
||||
return alloc::allocatedBytes();
|
||||
#endif
|
||||
}
|
||||
|
||||
void gc::GC::ClearForTests() noexcept {
|
||||
|
||||
@@ -21,14 +21,14 @@ public:
|
||||
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
|
||||
|
||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||
#endif
|
||||
SameThreadMarkAndSweep& gc() noexcept { return gc_; }
|
||||
|
||||
private:
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory objectFactory_;
|
||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||
#endif
|
||||
SameThreadMarkAndSweep gc_;
|
||||
};
|
||||
@@ -49,7 +49,7 @@ public:
|
||||
SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||
#else
|
||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||
#endif
|
||||
@@ -61,7 +61,7 @@ private:
|
||||
#else
|
||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#include "Logging.hpp"
|
||||
#include "MarkAndSweepUtils.hpp"
|
||||
#include "Memory.h"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "RootSet.hpp"
|
||||
#include "Runtime.h"
|
||||
#include "ThreadData.hpp"
|
||||
@@ -28,7 +27,7 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
|
||||
gcScheduler::GCScheduler& gcScheduler) noexcept :
|
||||
#else
|
||||
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
|
||||
ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
|
||||
ObjectFactory& objectFactory, alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
|
||||
|
||||
objectFactory_(objectFactory),
|
||||
extraObjectDataFactory_(extraObjectDataFactory),
|
||||
@@ -103,11 +102,11 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
||||
std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
|
||||
std::optional objectFactoryIterable = objectFactory_.LockForIter();
|
||||
|
||||
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||
extraObjectFactoryIterable = std::nullopt;
|
||||
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||
auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||
objectFactoryIterable = std::nullopt;
|
||||
kotlin::compactObjectPoolInMainThread();
|
||||
alloc::compactObjectPoolInMainThread();
|
||||
#else
|
||||
// also sweeps extraObjects
|
||||
auto finalizerQueue = heap_.Sweep(gcHandle);
|
||||
@@ -117,7 +116,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
||||
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
||||
#endif
|
||||
|
||||
scheduler.onGCFinish(epoch, allocatedBytes());
|
||||
scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
|
||||
|
||||
resumeTheWorld(gcHandle);
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ public:
|
||||
#else
|
||||
SameThreadMarkAndSweep(
|
||||
ObjectFactory& objectFactory,
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||
gcScheduler::GCScheduler& gcScheduler) noexcept;
|
||||
#endif
|
||||
|
||||
@@ -66,7 +66,7 @@ private:
|
||||
|
||||
#ifndef CUSTOM_ALLOCATOR
|
||||
ObjectFactory& objectFactory_;
|
||||
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||
alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||
#else
|
||||
alloc::Heap heap_;
|
||||
#endif
|
||||
|
||||
@@ -612,6 +612,9 @@ bool FinalizersThreadIsRunning() noexcept;
|
||||
|
||||
void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept;
|
||||
|
||||
void initObjectPool() noexcept;
|
||||
void compactObjectPoolInCurrentThread() noexcept;
|
||||
|
||||
} // namespace kotlin
|
||||
|
||||
RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object);
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
#include "MainQueueProcessor.hpp"
|
||||
#include "Memory.h"
|
||||
#include "ObjCExportInit.h"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "Porting.h"
|
||||
#include "Runtime.h"
|
||||
#include "RuntimePrivate.hpp"
|
||||
|
||||
@@ -32,7 +32,6 @@
|
||||
#include "Memory.h"
|
||||
#include "Natives.h"
|
||||
#include "ObjCMMAPI.h"
|
||||
#include "ObjectAlloc.hpp"
|
||||
#include "Runtime.h"
|
||||
#include "Types.h"
|
||||
#include "Worker.h"
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "Memory.h"
|
||||
#include "MemoryPrivate.hpp"
|
||||
|
||||
#include "Allocator.hpp"
|
||||
#include "Exceptions.h"
|
||||
#include "ExtraObjectData.hpp"
|
||||
#include "Freezing.hpp"
|
||||
@@ -636,3 +637,11 @@ RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakR
|
||||
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
|
||||
mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
|
||||
}
|
||||
|
||||
void kotlin::initObjectPool() noexcept {
|
||||
alloc::initObjectPool();
|
||||
}
|
||||
|
||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {
|
||||
alloc::compactObjectPoolInCurrentThread();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user