[K/N] Refactor allocator modules ^KT-60928
- Move std_alloc, opt_alloc and custom_alloc into alloc/std, alloc/mimalloc and alloc/custom to mirror convention of gc, gcScheduler modules. - Add alloc/common with common allocator API - Add alloc/legacy with common implementation details of alloc/std and alloc/mimalloc. alloc/custom does not depend on alloc/legacy. - Removes experimental_memory_manager_custom as it's now unused - Additionally, renames experimental_memory_manager module into mm
This commit is contained in:
Alexander Shabalin
committed by
Space Team
Space Team
parent
c6893de9bb
commit
1fa0ef6f56
+5
-3
@@ -298,6 +298,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
|||||||
|
|
||||||
internal val runtimeNativeLibraries: List<String> = mutableListOf<String>().apply {
|
internal val runtimeNativeLibraries: List<String> = mutableListOf<String>().apply {
|
||||||
if (debug) add("debug.bc")
|
if (debug) add("debug.bc")
|
||||||
|
add("mm.bc")
|
||||||
|
add("common_alloc.bc")
|
||||||
add("common_gc.bc")
|
add("common_gc.bc")
|
||||||
add("common_gcScheduler.bc")
|
add("common_gcScheduler.bc")
|
||||||
when (gcSchedulerType) {
|
when (gcSchedulerType) {
|
||||||
@@ -315,14 +317,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (allocationMode == AllocationMode.CUSTOM) {
|
if (allocationMode == AllocationMode.CUSTOM) {
|
||||||
add("experimental_memory_manager_custom.bc")
|
|
||||||
when (gc) {
|
when (gc) {
|
||||||
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc")
|
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc")
|
||||||
GC.NOOP -> add("noop_gc_custom.bc")
|
GC.NOOP -> add("noop_gc_custom.bc")
|
||||||
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc")
|
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
add("experimental_memory_manager.bc")
|
|
||||||
when (gc) {
|
when (gc) {
|
||||||
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc")
|
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc")
|
||||||
GC.NOOP -> add("noop_gc.bc")
|
GC.NOOP -> add("noop_gc.bc")
|
||||||
@@ -339,10 +339,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
|
|||||||
}
|
}
|
||||||
when (allocationMode) {
|
when (allocationMode) {
|
||||||
AllocationMode.MIMALLOC -> {
|
AllocationMode.MIMALLOC -> {
|
||||||
add("opt_alloc.bc")
|
add("legacy_alloc.bc")
|
||||||
|
add("mimalloc_alloc.bc")
|
||||||
add("mimalloc.bc")
|
add("mimalloc.bc")
|
||||||
}
|
}
|
||||||
AllocationMode.STD -> {
|
AllocationMode.STD -> {
|
||||||
|
add("legacy_alloc.bc")
|
||||||
add("std_alloc.bc")
|
add("std_alloc.bc")
|
||||||
}
|
}
|
||||||
AllocationMode.CUSTOM -> {
|
AllocationMode.CUSTOM -> {
|
||||||
|
|||||||
@@ -150,15 +150,25 @@ bitcode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module("common_alloc") {
|
||||||
|
srcRoot.set(layout.projectDirectory.dir("src/alloc/common"))
|
||||||
|
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
|
sourceSets {
|
||||||
|
main {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
module("std_alloc") {
|
module("std_alloc") {
|
||||||
headersDirs.from(files("src/main/cpp"))
|
srcRoot.set(layout.projectDirectory.dir("src/alloc/std"))
|
||||||
|
headersDirs.from(files("src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module("custom_alloc") {
|
module("custom_alloc") {
|
||||||
headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp"))
|
srcRoot.set(layout.projectDirectory.dir("src/alloc/custom"))
|
||||||
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
test {}
|
test {}
|
||||||
@@ -167,8 +177,9 @@ bitcode {
|
|||||||
compilerArgs.add("-DCUSTOM_ALLOCATOR")
|
compilerArgs.add("-DCUSTOM_ALLOCATOR")
|
||||||
}
|
}
|
||||||
|
|
||||||
module("opt_alloc") {
|
module("mimalloc_alloc") {
|
||||||
headersDirs.from(files("src/main/cpp"))
|
srcRoot.set(layout.projectDirectory.dir("src/alloc/mimalloc"))
|
||||||
|
headersDirs.from(files("src/mimalloc/c/include", "src/alloc/common/cpp", "src/alloc/legacy/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
}
|
}
|
||||||
@@ -176,6 +187,15 @@ bitcode {
|
|||||||
compilerArgs.add("-DKONAN_MI_MALLOC=1")
|
compilerArgs.add("-DKONAN_MI_MALLOC=1")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module("legacy_alloc") {
|
||||||
|
srcRoot.set(layout.projectDirectory.dir("src/alloc/legacy"))
|
||||||
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
|
sourceSets {
|
||||||
|
main {}
|
||||||
|
test {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
module("exceptionsSupport") {
|
module("exceptionsSupport") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/exceptions_support"))
|
srcRoot.set(layout.projectDirectory.dir("src/exceptions_support"))
|
||||||
headersDirs.from(files("src/main/cpp"))
|
headersDirs.from(files("src/main/cpp"))
|
||||||
@@ -227,9 +247,8 @@ bitcode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module("experimental_memory_manager") {
|
module("mm") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/mm"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp"))
|
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -237,21 +256,9 @@ bitcode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module("experimental_memory_manager_custom") {
|
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/mm"))
|
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
|
||||||
sourceSets {
|
|
||||||
main {}
|
|
||||||
testFixtures {}
|
|
||||||
test {}
|
|
||||||
}
|
|
||||||
|
|
||||||
compilerArgs.add("-DCUSTOM_ALLOCATOR")
|
|
||||||
}
|
|
||||||
|
|
||||||
module("common_gc") {
|
module("common_gc") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/common"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/common"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
test {}
|
test {}
|
||||||
@@ -260,7 +267,7 @@ bitcode {
|
|||||||
|
|
||||||
module("noop_gc") {
|
module("noop_gc") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -269,7 +276,7 @@ bitcode {
|
|||||||
|
|
||||||
module("noop_gc_custom") {
|
module("noop_gc_custom") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -280,7 +287,7 @@ bitcode {
|
|||||||
|
|
||||||
module("same_thread_ms_gc") {
|
module("same_thread_ms_gc") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -290,7 +297,7 @@ bitcode {
|
|||||||
|
|
||||||
module("same_thread_ms_gc_custom") {
|
module("same_thread_ms_gc_custom") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -302,7 +309,7 @@ bitcode {
|
|||||||
|
|
||||||
module("concurrent_ms_gc") {
|
module("concurrent_ms_gc") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -312,7 +319,7 @@ bitcode {
|
|||||||
|
|
||||||
module("concurrent_ms_gc_custom") {
|
module("concurrent_ms_gc_custom") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
testFixtures {}
|
testFixtures {}
|
||||||
@@ -324,7 +331,7 @@ bitcode {
|
|||||||
|
|
||||||
module("common_gcScheduler") {
|
module("common_gcScheduler") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/common"))
|
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/common"))
|
||||||
headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
test {}
|
test {}
|
||||||
@@ -333,7 +340,7 @@ bitcode {
|
|||||||
|
|
||||||
module("manual_gcScheduler") {
|
module("manual_gcScheduler") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/manual"))
|
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/manual"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
}
|
}
|
||||||
@@ -341,7 +348,7 @@ bitcode {
|
|||||||
|
|
||||||
module("adaptive_gcScheduler") {
|
module("adaptive_gcScheduler") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/adaptive"))
|
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/adaptive"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
test {}
|
test {}
|
||||||
@@ -350,7 +357,7 @@ bitcode {
|
|||||||
|
|
||||||
module("aggressive_gcScheduler") {
|
module("aggressive_gcScheduler") {
|
||||||
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/aggressive"))
|
srcRoot.set(layout.projectDirectory.dir("src/gcScheduler/aggressive"))
|
||||||
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
|
||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
test {}
|
test {}
|
||||||
@@ -359,56 +366,56 @@ bitcode {
|
|||||||
|
|
||||||
testsGroup("custom_alloc_runtime_tests") {
|
testsGroup("custom_alloc_runtime_tests") {
|
||||||
testedModules.addAll("custom_alloc")
|
testedModules.addAll("custom_alloc")
|
||||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "objc")
|
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc_custom", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_mimalloc_runtime_tests") {
|
testsGroup("experimentalMM_mimalloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "opt_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_std_alloc_runtime_tests") {
|
testsGroup("experimentalMM_std_alloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_custom_alloc_runtime_tests") {
|
testsGroup("experimentalMM_custom_alloc_runtime_tests") {
|
||||||
testedModules.addAll("experimental_memory_manager_custom", "same_thread_ms_gc_custom")
|
testedModules.addAll("mm", "same_thread_ms_gc_custom")
|
||||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_cms_mimalloc_runtime_tests") {
|
testsGroup("experimentalMM_cms_mimalloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_cms_std_alloc_runtime_tests") {
|
testsGroup("experimentalMM_cms_std_alloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") {
|
testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") {
|
||||||
testedModules.addAll("experimental_memory_manager_custom", "concurrent_ms_gc_custom")
|
testedModules.addAll("mm", "concurrent_ms_gc_custom")
|
||||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_noop_mimalloc_runtime_tests") {
|
testsGroup("experimentalMM_noop_mimalloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "mimalloc_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_noop_std_alloc_runtime_tests") {
|
testsGroup("experimentalMM_noop_std_alloc_runtime_tests") {
|
||||||
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc")
|
testedModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") {
|
testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") {
|
||||||
testedModules.addAll("experimental_memory_manager_custom", "noop_gc_custom")
|
testedModules.addAll("mm", "noop_gc_custom")
|
||||||
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
testSupportModules.addAll("main", "common_alloc", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("aggressive_gcScheduler_runtime_tests") {
|
testsGroup("aggressive_gcScheduler_runtime_tests") {
|
||||||
testedModules.addAll("aggressive_gcScheduler")
|
testedModules.addAll("aggressive_gcScheduler")
|
||||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc")
|
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
|
|
||||||
testsGroup("adaptive_gcScheduler_runtime_tests") {
|
testsGroup("adaptive_gcScheduler_runtime_tests") {
|
||||||
testedModules.addAll("adaptive_gcScheduler")
|
testedModules.addAll("adaptive_gcScheduler")
|
||||||
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc")
|
testSupportModules.addAll("main", "mm", "common_alloc", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "legacy_alloc", "objc")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||||
|
* that can be found in the LICENSE file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "Allocator.hpp"
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||||
|
* that can be found in the LICENSE file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
namespace kotlin::alloc {
|
||||||
|
|
||||||
|
// TODO: Build `Allocator`, `Allocator::ThreadData` like with `gc`, `gcScheduler`,
|
||||||
|
// and move allocator-specific data there.
|
||||||
|
|
||||||
|
void initObjectPool() noexcept;
|
||||||
|
void compactObjectPoolInCurrentThread() noexcept;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,372 @@
|
|||||||
|
# Overview of the allocator
|
||||||
|
|
||||||
|
This document describes the internals of the custom allocator. The presentation
|
||||||
|
here is not fully true to the implementation, as the design is still work in
|
||||||
|
progress.
|
||||||
|
|
||||||
|
The main idea of the custom allocator is to divide system memory into chunks
|
||||||
|
(pages) that each can be swept independently, in memory consecutive order.
|
||||||
|
Every allocation ends up as a block of memory inside a page. Each page keeps
|
||||||
|
track of the size of each block it contains; how this is done depends on the
|
||||||
|
page type, with different page types optimized for different allocation sizes.
|
||||||
|
All the memory blocks are consecutive within a page, so the size of a block
|
||||||
|
also tells where the next block begins. Paired with an additional mechanism
|
||||||
|
(page type dependent) for determining whether a block is allocated, we can
|
||||||
|
iterate through the allocated blocks.
|
||||||
|
|
||||||
|
When a thread allocates memory for an object, it will find a page suitable for
|
||||||
|
the given allocation size. Each thread holds on to a number of pages for the
|
||||||
|
different size categories. The typical case is that the thread’s current page
|
||||||
|
for the given size can fit the requested allocation. If that is not the case, a
|
||||||
|
different page for that size category is requested from the shared allocation
|
||||||
|
space. The requested page can either be readily available (already prepared by
|
||||||
|
the GC thread), or it might need to be swept first, or it might be newly
|
||||||
|
created.
|
||||||
|
|
||||||
|
The GC thread has a new responsibility when using this allocator: While the
|
||||||
|
mutator threads are paused at the start of GC, the GC thread must prepare the
|
||||||
|
allocator for sweeping. This does two things. First, it marks all pages as
|
||||||
|
“needs to be swept before next use”. Second, it releases pages that threads are
|
||||||
|
holding on to, by clearing the thread local variables for each thread.
|
||||||
|
|
||||||
|
It is possible to have several independent allocation spaces at the same time.
|
||||||
|
This is currently useful for testing, but is potentially useful in other
|
||||||
|
settings.
|
||||||
|
|
||||||
|
# Detailed Design
|
||||||
|
|
||||||
|
All allocations are made through a `CustomAllocator` object.
|
||||||
|
|
||||||
|
## [CustomAllocator](cpp/CustomAllocator.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class CustomAllocator {
|
||||||
|
public:
|
||||||
|
CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler);
|
||||||
|
ObjectHeader* CreateObject(TypeInfo* type);
|
||||||
|
ArrayHeader* CreateArray(TypeInfo* type, uint32_t count);
|
||||||
|
ExtraObjectData* CreateExtraObject();
|
||||||
|
void PrepareForGc();
|
||||||
|
|
||||||
|
private:
|
||||||
|
uint8_t* Allocate(uint64_t cellCount);
|
||||||
|
uint8_t* AllocateInSingleObjectPage(uint64_t cellCount);
|
||||||
|
uint8_t* AllocateInNextFitPage(uint32_t cellCount);
|
||||||
|
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount);
|
||||||
|
|
||||||
|
Heap& heap_;
|
||||||
|
GCSchedulerThreadData& gcScheduler_;
|
||||||
|
NextFitPage* nextFitPage_;
|
||||||
|
FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||||
|
ExtraObjectPage* extraObjectPage_;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
The primary responsibility of this class is to delegate each requested
|
||||||
|
allocation to pages of the appropriate type, based on allocation size. To do
|
||||||
|
this, it requests pages from the shared allocation space (`Heap`) and stores
|
||||||
|
pages for later allocations. Each thread thus owns a number of pages for
|
||||||
|
different allocation sizes, but at most one for each size class. When
|
||||||
|
allocating, the `CustomAllocator` will first try to allocate in one of its
|
||||||
|
owned pages. If this fails, it will request a new page for that size class from
|
||||||
|
a shared `Heap` object. `SingleObjectPages` are never kept by the
|
||||||
|
`CustomAllocator`, since they are created specifically for a single allocation,
|
||||||
|
with no extra space.
|
||||||
|
|
||||||
|
## [Heap](cpp/Heap.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class Heap {
|
||||||
|
public:
|
||||||
|
void PrepareForGC();
|
||||||
|
|
||||||
|
void Sweep();
|
||||||
|
|
||||||
|
AtomicStack<ExtraObjectCell> SweepExtraObjects(GCHandle gcHandle);
|
||||||
|
|
||||||
|
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount);
|
||||||
|
NextFitPage* GetNextFitPage(uint32_t cellCount);
|
||||||
|
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount);
|
||||||
|
ExtraObjectPage* GetExtraObjectPage();
|
||||||
|
|
||||||
|
private:
|
||||||
|
PageStore<FixedBlockPage> fixedBlockPages_[MAX_BLOCK_SIZE];
|
||||||
|
PageStore<NextFitPage> nextFitPages_;
|
||||||
|
PageStore<SingleObjectPage> singleObjectPages_;
|
||||||
|
AtomicStack<ExtraObjectPage> extraObjectPages_;
|
||||||
|
AtomicStack<ExtraObjectPage> usedExtraObjectPages_;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
A `Heap` object represents a shared allocation space for multiple
|
||||||
|
`CustomAllocator`s, which can request pages through one of the
|
||||||
|
`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also
|
||||||
|
provides a method for sweeping through all blocks that have been allocated in
|
||||||
|
this heap. The `Heap` object is the synchronization point, and guarantees that
|
||||||
|
every page is returned at most once. Page ownership is thus implicitly given to
|
||||||
|
the thread that called the method. The `Heap` object keeps track of all pages,
|
||||||
|
so there is no need to explicitly return ownership of a page. Internally, a
|
||||||
|
`Heap` keeps the pages for each size class in a `PageStore`. This means one for
|
||||||
|
`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes
|
||||||
|
for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two
|
||||||
|
`AtomicStack`s, since they require different handling during sweeping.
|
||||||
|
|
||||||
|
## [PageStore](cpp/PageStore.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
template <class PageType>
|
||||||
|
class PageStore {
|
||||||
|
public:
|
||||||
|
void PrepareForGC();
|
||||||
|
void Sweep();
|
||||||
|
void SweepAndFree();
|
||||||
|
PageType* GetPage(uint32_t cellCount);
|
||||||
|
PageType* NewPage(uint64_t cellCount);
|
||||||
|
|
||||||
|
private:
|
||||||
|
AtomicStack<PageType> empty_;
|
||||||
|
AtomicStack<PageType> ready_;
|
||||||
|
AtomicStack<PageType> used_;
|
||||||
|
AtomicStack<PageType> unswept_;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
A PageStore is responsible for keeping track of all pages of a given type and
|
||||||
|
size class. Each of the pages are in one of four stacks. The stack, that a
|
||||||
|
given page is in, determines its current state:
|
||||||
|
|
||||||
|
* `unswept_`: have not yet been swept since the last GC cycle.
|
||||||
|
* `ready_`: are ready for allocation; has been swept by the GC thread.
|
||||||
|
* `used_`: has been given to some thread for allocation; it might still be used
|
||||||
|
for allocation, or it might have been discarded with not enough space left.
|
||||||
|
Will not be used until the next GC cycle.
|
||||||
|
* `empty_`: same as `ready_`, but does not contain any objects. Will be freed
|
||||||
|
before the next GC, if not needed before then.
|
||||||
|
|
||||||
|
When a page is requested, the page is taken from `ready_`, if there are any.
|
||||||
|
Otherwise, an `unswept_` page is taken and swept before returning. If there are
|
||||||
|
no unswept pages either, an empty page is taken, if there are any. Otherwise a
|
||||||
|
new page is created in the size category. All returned pages are moved to
|
||||||
|
`used_`. During the marking phase, all remaining pages in `empty_` are freed,
|
||||||
|
and all other pages are moved to `unswept_`. The GC thread will go through all
|
||||||
|
`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one
|
||||||
|
of the other threads sweeps a page from `unswept_`, it is moved directly to
|
||||||
|
`used_`, as it is claimed by the `CustomAllocator` that swept it.
|
||||||
|
|
||||||
|
`SingleObjectPage`s are treated slightly differently, because they are created
|
||||||
|
for one specific single allocation, and not reused when that allocation is
|
||||||
|
freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without
|
||||||
|
checking any of the stacks, and during sweeping, they are freed directly rather
|
||||||
|
than being put into the `empty_` stack. Ideally, there would only be two stacks
|
||||||
|
in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little
|
||||||
|
is lost by just using the existing `PageStore` logic used for the other pages.
|
||||||
|
|
||||||
|
## [AtomicStack](cpp/AtomicStack.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
template <class PageType>
|
||||||
|
class AtomicStack {
|
||||||
|
public:
|
||||||
|
PageType* Pop();
|
||||||
|
void Push(PageType* elm);
|
||||||
|
void TransferAllFrom(AtomicStack<T>& src);
|
||||||
|
bool isEmpty();
|
||||||
|
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::atomic<PageType*> stack_;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
The only place where atomics are used are in the stacks inside the `PageStore`.
|
||||||
|
All page classes have a non-atomic next pointer, to be used for linking up in
|
||||||
|
exactly one stack. `Pop` and `Push` are implemented with compare-and-swap
|
||||||
|
operations. The class is thread safe, except for if an element is freed while
|
||||||
|
another thread tries to Pop it from a stack.
|
||||||
|
|
||||||
|
# Page types
|
||||||
|
|
||||||
|
This section is likely to change, given the likely introduction of additional
|
||||||
|
page types. It also describes some details about which page type is chosen for
|
||||||
|
a given allocation, which is also likely to change.
|
||||||
|
|
||||||
|
There are four different page types, but they all share the feature that they
|
||||||
|
can be swept independently. The Sweep methods return whether there were any
|
||||||
|
live objects in the page after sweeping. If not, the page will be given back to
|
||||||
|
the OS.
|
||||||
|
|
||||||
|
## [FixedBlockPage](cpp/FixedBlockPage.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class FixedBlockPage {
|
||||||
|
public:
|
||||||
|
FixedBlockPage(uint32_t blockSize) noexcept;
|
||||||
|
|
||||||
|
uint8_t* TryAllocate() noexcept;
|
||||||
|
|
||||||
|
bool Sweep() noexcept;
|
||||||
|
|
||||||
|
private:
|
||||||
|
FixedBlockPage* next_;
|
||||||
|
FixedCellRange nextFree_;
|
||||||
|
uint32_t blockSize_;
|
||||||
|
uint32_t end_;
|
||||||
|
FixedBlockCell cells_[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
All sufficiently small allocations (currently arbitrary <1KiB) are directed to
|
||||||
|
a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations
|
||||||
|
are expected to be in this page type. A `FixedBlockPage` consists of a number
|
||||||
|
of equally sized blocks, where each allocation will take up exactly one such
|
||||||
|
block.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
struct FixedBlockCell {
|
||||||
|
union {
|
||||||
|
uint8_t data[];
|
||||||
|
FixedCellRange nextFree;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct alignas(8) FixedCellRange {
|
||||||
|
uint32_t first;
|
||||||
|
uint32_t last;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Consecutive unallocated cells are represented by a `FixedCellRange`, with
|
||||||
|
`.first` and `.last` being the inclusive end points of the range of unallocated
|
||||||
|
cells. The `FixedBlockCell` at the the `.last` index contains a
|
||||||
|
`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange`
|
||||||
|
of unallocated ranges thus form a linked list.
|
||||||
|
|
||||||
|
The important point is that all links in this list point forward in the page, so
|
||||||
|
all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a
|
||||||
|
`FixedBlockPage` consists of walking the free-list forward, and sweeping all
|
||||||
|
blocks in between the links, maintaining the free list when blocks are freed.
|
||||||
|
|
||||||
|
Each small page takes up the same amount of space, independent of block size,
|
||||||
|
so larger block size implies fewer blocks per page. This size is arbitrarily
|
||||||
|
chosen to be 256 KiB, but this might change.
|
||||||
|
|
||||||
|
## [NextFitPage](cpp/NextFitPage.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class NextFitPage {
|
||||||
|
public:
|
||||||
|
NextFitPage(uint32_t cellCount);
|
||||||
|
Cell* TryAllocate(uint32_t cellCount);
|
||||||
|
bool Sweep();
|
||||||
|
|
||||||
|
private:
|
||||||
|
NextFitPage* next_; // used by AtomicStack
|
||||||
|
Cell* curBlock_;
|
||||||
|
Cell cells_[];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Allocations that could theoretically fit in a `FixedBlockPage`, but would
|
||||||
|
require too large a block size (arbitrary >=1KiB), are allocated in a
|
||||||
|
`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64
|
||||||
|
KiB for experiments). All blocks in a `NextFitPage` have a header that tells how
|
||||||
|
big the block is, and whether it is allocated or not. There are no gaps between
|
||||||
|
blocks, so the size of a block also tells where the next block is. The header
|
||||||
|
information fits inside a 8 byte `Cell`.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class Cell {
|
||||||
|
public:
|
||||||
|
Cell(uint32_t size);
|
||||||
|
uint8_t* TryAllocate(uint32_t cellCount);
|
||||||
|
|
||||||
|
private:
|
||||||
|
uint32_t isAllocated_;
|
||||||
|
uint32_t size_;
|
||||||
|
uint8_t data_[];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
The page keeps a reference to a currently active block, and will try to bump
|
||||||
|
allocate inside that block. If allocation does not fit, we move to the next
|
||||||
|
block that fits. If no block in the page fits the requested size, the page is
|
||||||
|
abandoned until the next GC.
|
||||||
|
|
||||||
|
## [SingleObjectPage](cpp/SingleObjectPage.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class SingleObjectPage {
|
||||||
|
public:
|
||||||
|
SingleObjectPage(uint64_t cellCount);
|
||||||
|
bool Sweep();
|
||||||
|
|
||||||
|
private:
|
||||||
|
SingleObjectPage* next_; // used by AtomicStack
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`,
|
||||||
|
which only contains that single block of the requested size. They are also
|
||||||
|
handled slightly differently by both `Heap` and `CustomAllocator`. First off,
|
||||||
|
`Heap::GetSingleObjectPage` will never check existing pages, and instead just
|
||||||
|
allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to
|
||||||
|
any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC
|
||||||
|
thread.
|
||||||
|
|
||||||
|
## [ExtraObjectPage](cpp/ExtraObjectPage.hpp)
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class ExtraObjectPage {
|
||||||
|
public:
|
||||||
|
ExtraObjectPage();
|
||||||
|
ExtraObjectData* TryAllocate();
|
||||||
|
bool Sweep(FinalizerQueue& queue);
|
||||||
|
|
||||||
|
|
||||||
|
private:
|
||||||
|
ExtraObjectPage* next_; // used by AtomicStack
|
||||||
|
ExtraObjectCell* nextFree_;
|
||||||
|
ExtraObjectCell cells_[];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Extra objects are used for attaching additional data to some objects. This is
|
||||||
|
used for objects that require special handling during garbage collection:
|
||||||
|
|
||||||
|
* objects with finalizers
|
||||||
|
* weak references
|
||||||
|
* interop references
|
||||||
|
|
||||||
|
Extra objects are allocated in `ExtraObjectPage`s, which are very similar to
|
||||||
|
`FixedBlockPage`s. They primarily differ in how they are swept, since it is
|
||||||
|
during sweeping of `ExtraObject`s that scheduling of finalization happens. If
|
||||||
|
an object that requires finalization is found, it is added to the
|
||||||
|
`FinalizerQueue` given as argument. The cells are also slightly different, in
|
||||||
|
that they add a new field that allows the cells to be added to the finalizer
|
||||||
|
queue.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
struct ExtraObjectCell {
|
||||||
|
ExtraObjectCell* next_; // used by AtomicStack
|
||||||
|
ExtraObjectData data_;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
# Finalizers
|
||||||
|
|
||||||
|
Section like to change.
|
||||||
|
|
||||||
|
In the existing memory model, finalization tasks are found and scheduled during
|
||||||
|
sweeping of regular objects. The objects to be finalized are chained together
|
||||||
|
using a pointer in the Node header, added to all allocated objects. This header
|
||||||
|
is not needed in the custom allocator, apart from linking in the finalization
|
||||||
|
queue.
|
||||||
|
|
||||||
|
We therefore reintroduce this pointer in a header for `ExtraObjectData`. For
|
||||||
|
this, we reuse the `ExtraObjectCell` as header for both free list pointer and as
|
||||||
|
linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`.
|
||||||
|
|
||||||
|
# Enabling the allocator
|
||||||
|
|
||||||
|
The custom allocator is enabled with the compiler flag -Xallocator=custom.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||||
|
* that can be found in the LICENSE file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "Allocator.hpp"
|
||||||
|
|
||||||
|
using namespace kotlin;
|
||||||
|
|
||||||
|
void alloc::initObjectPool() noexcept {}
|
||||||
|
|
||||||
|
void alloc::compactObjectPoolInCurrentThread() noexcept {}
|
||||||
+4
-4
@@ -10,14 +10,14 @@
|
|||||||
|
|
||||||
using namespace kotlin;
|
using namespace kotlin;
|
||||||
|
|
||||||
mm::ExtraObjectData& mm::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject(
|
mm::ExtraObjectData& alloc::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject(
|
||||||
ObjHeader* baseObject, const TypeInfo* info) noexcept {
|
ObjHeader* baseObject, const TypeInfo* info) noexcept {
|
||||||
return **Emplace(baseObject, info);
|
return **Emplace(baseObject, info);
|
||||||
}
|
}
|
||||||
|
|
||||||
void mm::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(ExtraObjectData& data) noexcept {
|
void alloc::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept {
|
||||||
Erase(&Queue::Node::fromValue(data));
|
Erase(&Queue::Node::fromValue(data));
|
||||||
}
|
}
|
||||||
|
|
||||||
mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
|
alloc::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
|
||||||
mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
|
alloc::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
|
||||||
+5
-10
@@ -3,8 +3,7 @@
|
|||||||
* that can be found in the LICENSE file.
|
* that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
#pragma once
|
||||||
#define RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
|
||||||
|
|
||||||
#include "ExtraObjectData.hpp"
|
#include "ExtraObjectData.hpp"
|
||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
@@ -12,8 +11,7 @@
|
|||||||
#include "ObjectAlloc.hpp"
|
#include "ObjectAlloc.hpp"
|
||||||
#include "ThreadRegistry.hpp"
|
#include "ThreadRegistry.hpp"
|
||||||
|
|
||||||
namespace kotlin {
|
namespace kotlin::alloc {
|
||||||
namespace mm {
|
|
||||||
|
|
||||||
// Registry for extra data, attached to some kotlin objects: weak refs, associated objects, ...
|
// Registry for extra data, attached to some kotlin objects: weak refs, associated objects, ...
|
||||||
class ExtraObjectDataFactory : Pinned {
|
class ExtraObjectDataFactory : Pinned {
|
||||||
@@ -24,9 +22,9 @@ public:
|
|||||||
public:
|
public:
|
||||||
explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {}
|
explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {}
|
||||||
|
|
||||||
ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept;
|
mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept;
|
||||||
|
|
||||||
void DestroyExtraObjectData(ExtraObjectData& data) noexcept;
|
void DestroyExtraObjectData(mm::ExtraObjectData& data) noexcept;
|
||||||
|
|
||||||
// Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread
|
// Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread
|
||||||
// when it's asked by GC to stop.
|
// when it's asked by GC to stop.
|
||||||
@@ -56,7 +54,4 @@ private:
|
|||||||
Queue extraObjects_;
|
Queue extraObjects_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace mm
|
} // namespace kotlin::alloc
|
||||||
} // namespace kotlin
|
|
||||||
|
|
||||||
#endif // RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H
|
|
||||||
+1
-1
@@ -8,7 +8,7 @@
|
|||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <type_traits>
|
#include <type_traits>
|
||||||
|
|
||||||
namespace kotlin {
|
namespace kotlin::alloc {
|
||||||
|
|
||||||
void initObjectPool() noexcept;
|
void initObjectPool() noexcept;
|
||||||
void* allocateInObjectPool(size_t size) noexcept;
|
void* allocateInObjectPool(size_t size) noexcept;
|
||||||
+3
-8
@@ -3,8 +3,7 @@
|
|||||||
* that can be found in the LICENSE file.
|
* that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#ifndef RUNTIME_MM_OBJECT_FACTORY_H
|
#pragma once
|
||||||
#define RUNTIME_MM_OBJECT_FACTORY_H
|
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cinttypes>
|
#include <cinttypes>
|
||||||
@@ -21,8 +20,7 @@
|
|||||||
#include "Types.h"
|
#include "Types.h"
|
||||||
#include "Utils.hpp"
|
#include "Utils.hpp"
|
||||||
|
|
||||||
namespace kotlin {
|
namespace kotlin::alloc {
|
||||||
namespace mm {
|
|
||||||
|
|
||||||
namespace internal {
|
namespace internal {
|
||||||
|
|
||||||
@@ -736,7 +734,4 @@ private:
|
|||||||
Storage storage_;
|
Storage storage_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace mm
|
} // namespace kotlin::alloc
|
||||||
} // namespace kotlin
|
|
||||||
|
|
||||||
#endif // RUNTIME_MM_OBJECT_FACTORY_H
|
|
||||||
+3
-5
@@ -9,14 +9,13 @@
|
|||||||
|
|
||||||
#include "ObjectAlloc.hpp"
|
#include "ObjectAlloc.hpp"
|
||||||
|
|
||||||
namespace kotlin {
|
namespace kotlin::alloc {
|
||||||
namespace gc {
|
|
||||||
|
|
||||||
// TODO: Try to move from custom allocator interface to standard one.
|
// TODO: Try to move from custom allocator interface to standard one.
|
||||||
// Currently Free method is in the way: it is static to avoid keeping allocator state in
|
// Currently Free method is in the way: it is static to avoid keeping allocator state in
|
||||||
// unique_ptr's deleter in ObjectFactory.
|
// unique_ptr's deleter in ObjectFactory.
|
||||||
|
|
||||||
class Allocator {
|
class AllocatorBasic {
|
||||||
public:
|
public:
|
||||||
void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); }
|
void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); }
|
||||||
static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); }
|
static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); }
|
||||||
@@ -43,5 +42,4 @@ private:
|
|||||||
GCThreadData& gc_;
|
GCThreadData& gc_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace gc
|
} // namespace kotlin::alloc
|
||||||
} // namespace kotlin
|
|
||||||
+4
-4
@@ -3,7 +3,7 @@
|
|||||||
* that can be found in the LICENSE file.
|
* that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
|
|
||||||
#include "gmock/gmock.h"
|
#include "gmock/gmock.h"
|
||||||
#include "gtest/gtest.h"
|
#include "gtest/gtest.h"
|
||||||
@@ -48,7 +48,7 @@ TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
|
|||||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
||||||
EXPECT_CALL(gc, OnOOM(_)).Times(0);
|
EXPECT_CALL(gc, OnOOM(_)).Times(0);
|
||||||
}
|
}
|
||||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||||
void* ptr = allocator.Alloc(size);
|
void* ptr = allocator.Alloc(size);
|
||||||
EXPECT_THAT(ptr, nonNull);
|
EXPECT_THAT(ptr, nonNull);
|
||||||
}
|
}
|
||||||
@@ -64,7 +64,7 @@ TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
|
|||||||
EXPECT_CALL(gc, OnOOM(size));
|
EXPECT_CALL(gc, OnOOM(size));
|
||||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull));
|
||||||
}
|
}
|
||||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||||
void* ptr = allocator.Alloc(size);
|
void* ptr = allocator.Alloc(size);
|
||||||
EXPECT_THAT(ptr, nonNull);
|
EXPECT_THAT(ptr, nonNull);
|
||||||
}
|
}
|
||||||
@@ -79,7 +79,7 @@ TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
|
|||||||
EXPECT_CALL(gc, OnOOM(size));
|
EXPECT_CALL(gc, OnOOM(size));
|
||||||
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
|
EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr));
|
||||||
}
|
}
|
||||||
gc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
alloc::AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
|
||||||
void* ptr = allocator.Alloc(size);
|
void* ptr = allocator.Alloc(size);
|
||||||
EXPECT_THAT(ptr, nullptr);
|
EXPECT_THAT(ptr, nullptr);
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||||
|
* that can be found in the LICENSE file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "FinalizerHooks.hpp"
|
||||||
|
#include "GC.hpp"
|
||||||
|
#include "GCStatistics.hpp"
|
||||||
|
#include "ExtraObjectDataFactory.hpp"
|
||||||
|
|
||||||
|
namespace kotlin::alloc {
|
||||||
|
|
||||||
|
template <typename Traits>
|
||||||
|
void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept {
|
||||||
|
auto sweepHandle = handle.sweepExtraObjects();
|
||||||
|
factoryIter.ApplyDeletions();
|
||||||
|
for (auto it = factoryIter.begin(); it != factoryIter.end();) {
|
||||||
|
auto &extraObject = *it;
|
||||||
|
if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) {
|
||||||
|
extraObject.ClearRegularWeakReferenceImpl();
|
||||||
|
if (extraObject.HasAssociatedObject()) {
|
||||||
|
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
|
||||||
|
++it;
|
||||||
|
sweepHandle.addKeptObject();
|
||||||
|
} else {
|
||||||
|
extraObject.Uninstall();
|
||||||
|
it.EraseAndAdvance();
|
||||||
|
sweepHandle.addSweptObject();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
++it;
|
||||||
|
sweepHandle.addKeptObject();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Traits>
|
||||||
|
void SweepExtraObjects(gc::GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept {
|
||||||
|
auto iter = factory.LockForIter();
|
||||||
|
return SweepExtraObjects<Traits>(handle, iter);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Traits>
|
||||||
|
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
|
||||||
|
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
|
||||||
|
auto sweepHandle = handle.sweep();
|
||||||
|
|
||||||
|
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
|
||||||
|
if (Traits::TryResetMark(*it)) {
|
||||||
|
++it;
|
||||||
|
sweepHandle.addKeptObject();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
sweepHandle.addSweptObject();
|
||||||
|
auto* objHeader = it->GetObjHeader();
|
||||||
|
if (HasFinalizers(objHeader)) {
|
||||||
|
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
|
||||||
|
} else {
|
||||||
|
objectFactoryIter.EraseAndAdvance(it);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return finalizerQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename Traits>
|
||||||
|
typename Traits::ObjectFactory::FinalizerQueue Sweep(gc::GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
|
||||||
|
auto iter = objectFactory.LockForIter();
|
||||||
|
return Sweep<Traits>(handle, iter);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
struct DefaultSweepTraits {
|
||||||
|
using ObjectFactory = T;
|
||||||
|
using ExtraObjectsFactory = ExtraObjectDataFactory;
|
||||||
|
|
||||||
|
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
|
||||||
|
auto* baseObject = object.GetBaseObject();
|
||||||
|
if (!baseObject->heap()) return true;
|
||||||
|
return gc::isMarked(baseObject);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
+14
-13
@@ -3,17 +3,18 @@
|
|||||||
* that can be found in the LICENSE file.
|
* that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <TestSupport.hpp>
|
#include "ObjectFactorySweep.hpp"
|
||||||
#include "MarkAndSweepUtils.hpp"
|
|
||||||
|
|
||||||
#include "gmock/gmock.h"
|
#include "gmock/gmock.h"
|
||||||
#include "gtest/gtest.h"
|
#include "gtest/gtest.h"
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
|
||||||
#include "FinalizerHooksTestSupport.hpp"
|
|
||||||
#include "ObjectFactory.hpp"
|
|
||||||
#include "ObjectTestSupport.hpp"
|
|
||||||
#include "ExtraObjectDataFactory.hpp"
|
#include "ExtraObjectDataFactory.hpp"
|
||||||
|
#include "FinalizerHooksTestSupport.hpp"
|
||||||
|
#include "MarkAndSweepUtils.hpp"
|
||||||
|
#include "ObjectFactory.hpp"
|
||||||
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
|
#include "ObjectTestSupport.hpp"
|
||||||
|
#include "TestSupport.hpp"
|
||||||
#include "WeakRef.hpp"
|
#include "WeakRef.hpp"
|
||||||
|
|
||||||
using namespace kotlin;
|
using namespace kotlin;
|
||||||
@@ -45,11 +46,11 @@ struct ObjectFactoryTraits {
|
|||||||
State state = State::kUnmarked;
|
State state = State::kUnmarked;
|
||||||
};
|
};
|
||||||
|
|
||||||
using Allocator = gc::Allocator;
|
using Allocator = alloc::AllocatorBasic;
|
||||||
};
|
};
|
||||||
|
|
||||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||||
using ExtraObjectsDataFactory = mm::ExtraObjectDataFactory;
|
using ExtraObjectsDataFactory = alloc::ExtraObjectDataFactory;
|
||||||
|
|
||||||
class Object : public test_support::Object<Payload> {
|
class Object : public test_support::Object<Payload> {
|
||||||
public:
|
public:
|
||||||
@@ -134,7 +135,7 @@ ObjectFactoryTraits::ObjectData::State GetWeakReferenceState(test_support::Regul
|
|||||||
|
|
||||||
struct SweepTraits {
|
struct SweepTraits {
|
||||||
using ObjectFactory = ObjectFactory;
|
using ObjectFactory = ObjectFactory;
|
||||||
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
|
using ExtraObjectsFactory = alloc::ExtraObjectDataFactory;
|
||||||
|
|
||||||
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
|
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
|
||||||
auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).ObjectData();
|
auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).ObjectData();
|
||||||
@@ -195,8 +196,8 @@ public:
|
|||||||
|
|
||||||
std_support::vector<ObjHeader*> Sweep() {
|
std_support::vector<ObjHeader*> Sweep() {
|
||||||
gc::processWeaks<ProcessWeakTraits>(gc::GCHandle::getByEpoch(0), specialRefRegistry_);
|
gc::processWeaks<ProcessWeakTraits>(gc::GCHandle::getByEpoch(0), specialRefRegistry_);
|
||||||
gc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
|
alloc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
|
||||||
auto finalizers = gc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
|
auto finalizers = alloc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
|
||||||
std_support::vector<ObjHeader*> objects;
|
std_support::vector<ObjHeader*> objects;
|
||||||
for (auto node : finalizers.IterForTests()) {
|
for (auto node : finalizers.IterForTests()) {
|
||||||
objects.push_back(node.GetObjHeader());
|
objects.push_back(node.GetObjHeader());
|
||||||
@@ -266,7 +267,7 @@ private:
|
|||||||
kotlin::ScopedMemoryInit memoryInit;
|
kotlin::ScopedMemoryInit memoryInit;
|
||||||
FinalizerHooksTestSupport finalizerHooks_;
|
FinalizerHooksTestSupport finalizerHooks_;
|
||||||
ObjectFactory objectFactory_;
|
ObjectFactory objectFactory_;
|
||||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gc::Allocator()};
|
ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, alloc::AllocatorBasic()};
|
||||||
ExtraObjectsDataFactory extraObjectFactory_;
|
ExtraObjectsDataFactory extraObjectFactory_;
|
||||||
ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_};
|
ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_};
|
||||||
mm::SpecialRefRegistry specialRefRegistry_;
|
mm::SpecialRefRegistry specialRefRegistry_;
|
||||||
+3
-3
@@ -11,8 +11,8 @@
|
|||||||
#include "gmock/gmock.h"
|
#include "gmock/gmock.h"
|
||||||
#include "gtest/gtest.h"
|
#include "gtest/gtest.h"
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
|
||||||
#include "FinalizerHooksTestSupport.hpp"
|
#include "FinalizerHooksTestSupport.hpp"
|
||||||
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
#include "ObjectOps.hpp"
|
#include "ObjectOps.hpp"
|
||||||
#include "ObjectTestSupport.hpp"
|
#include "ObjectTestSupport.hpp"
|
||||||
#include "ScopedThread.hpp"
|
#include "ScopedThread.hpp"
|
||||||
@@ -39,7 +39,7 @@ struct DataSizeProvider {
|
|||||||
};
|
};
|
||||||
|
|
||||||
template <size_t DataAlignment>
|
template <size_t DataAlignment>
|
||||||
using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator, DataSizeProvider>;
|
using ObjectFactoryStorage = alloc::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator, DataSizeProvider>;
|
||||||
|
|
||||||
using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
|
using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
|
||||||
|
|
||||||
@@ -848,7 +848,7 @@ public:
|
|||||||
using Allocator = GlobalMockAllocator;
|
using Allocator = GlobalMockAllocator;
|
||||||
};
|
};
|
||||||
|
|
||||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||||
|
|
||||||
struct Payload {
|
struct Payload {
|
||||||
ObjHeader* field1;
|
ObjHeader* field1;
|
||||||
+8
-8
@@ -7,10 +7,10 @@
|
|||||||
|
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
|
||||||
#include "../../mimalloc/c/include/mimalloc.h"
|
|
||||||
#include "Alignment.hpp"
|
#include "Alignment.hpp"
|
||||||
#include "CompilerConstants.hpp"
|
#include "CompilerConstants.hpp"
|
||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
|
#include "mimalloc.h"
|
||||||
|
|
||||||
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
||||||
#include <dispatch/dispatch.h>
|
#include <dispatch/dispatch.h>
|
||||||
@@ -28,7 +28,7 @@ std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT;
|
|||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
void kotlin::initObjectPool() noexcept {
|
void alloc::initObjectPool() noexcept {
|
||||||
if (!compiler::mimallocUseDefaultOptions()) {
|
if (!compiler::mimallocUseDefaultOptions()) {
|
||||||
std::call_once(initOptions, [] {
|
std::call_once(initOptions, [] {
|
||||||
mi_option_enable(mi_option_reset_decommits);
|
mi_option_enable(mi_option_reset_decommits);
|
||||||
@@ -40,20 +40,20 @@ void kotlin::initObjectPool() noexcept {
|
|||||||
mi_thread_init();
|
mi_thread_init();
|
||||||
}
|
}
|
||||||
|
|
||||||
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
void* alloc::allocateInObjectPool(size_t size) noexcept {
|
||||||
return mi_calloc_aligned(1, size, kObjectAlignment);
|
return mi_calloc_aligned(1, size, kObjectAlignment);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
void alloc::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||||
mi_free(ptr);
|
mi_free(ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {
|
void alloc::compactObjectPoolInCurrentThread() noexcept {
|
||||||
if (!compiler::mimallocUseCompaction()) return;
|
if (!compiler::mimallocUseCompaction()) return;
|
||||||
mi_collect(true);
|
mi_collect(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInMainThread() noexcept {
|
void alloc::compactObjectPoolInMainThread() noexcept {
|
||||||
if (!compiler::mimallocUseCompaction()) return;
|
if (!compiler::mimallocUseCompaction()) return;
|
||||||
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
|
||||||
if (scheduledCompactOnMainThread.test_and_set()) {
|
if (scheduledCompactOnMainThread.test_and_set()) {
|
||||||
@@ -62,14 +62,14 @@ void kotlin::compactObjectPoolInMainThread() noexcept {
|
|||||||
}
|
}
|
||||||
dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) {
|
dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) {
|
||||||
if (mm::IsCurrentThreadRegistered()) {
|
if (mm::IsCurrentThreadRegistered()) {
|
||||||
kotlin::compactObjectPoolInCurrentThread();
|
alloc::compactObjectPoolInCurrentThread();
|
||||||
}
|
}
|
||||||
scheduledCompactOnMainThread.clear();
|
scheduledCompactOnMainThread.clear();
|
||||||
});
|
});
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t kotlin::allocatedBytes() noexcept {
|
size_t alloc::allocatedBytes() noexcept {
|
||||||
return mi_allocated_size();
|
return mi_allocated_size();
|
||||||
}
|
}
|
||||||
|
|
||||||
+6
-6
@@ -36,9 +36,9 @@ size_t allocatedBytesCounter = 0;
|
|||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
void kotlin::initObjectPool() noexcept {}
|
void alloc::initObjectPool() noexcept {}
|
||||||
|
|
||||||
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
void* alloc::allocateInObjectPool(size_t size) noexcept {
|
||||||
// TODO: Check that alignment to kObjectAlignment is satisfied.
|
// TODO: Check that alignment to kObjectAlignment is satisfied.
|
||||||
void* result = callocImpl(1, size);
|
void* result = callocImpl(1, size);
|
||||||
#ifndef KONAN_NO_THREADS
|
#ifndef KONAN_NO_THREADS
|
||||||
@@ -52,7 +52,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
void alloc::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||||
#ifndef KONAN_NO_THREADS
|
#ifndef KONAN_NO_THREADS
|
||||||
allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed);
|
allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed);
|
||||||
#else
|
#else
|
||||||
@@ -61,11 +61,11 @@ void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
|||||||
freeImpl(ptr);
|
freeImpl(ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {}
|
void alloc::compactObjectPoolInCurrentThread() noexcept {}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInMainThread() noexcept {}
|
void alloc::compactObjectPoolInMainThread() noexcept {}
|
||||||
|
|
||||||
size_t kotlin::allocatedBytes() noexcept {
|
size_t alloc::allocatedBytes() noexcept {
|
||||||
#ifndef KONAN_NO_THREADS
|
#ifndef KONAN_NO_THREADS
|
||||||
return allocatedBytesCounter.load();
|
return allocatedBytesCounter.load();
|
||||||
#else
|
#else
|
||||||
@@ -1,372 +1 @@
|
|||||||
# Overview of the allocator
|
Allocator documentation was moved [here](../alloc/custom/README.md).
|
||||||
|
|
||||||
This document describes the internals of the custom allocator. The presentation
|
|
||||||
here is not fully true to the implementation, as the design is still work in
|
|
||||||
progress.
|
|
||||||
|
|
||||||
The main idea of the custom allocator is to divide system memory into chunks
|
|
||||||
(pages) that each can be swept independently, in memory consecutive order.
|
|
||||||
Every allocation ends up as a block of memory inside a page. Each page keeps
|
|
||||||
track of the size of each block it contains; how this is done depends on the
|
|
||||||
page type, with different page types optimized for different allocation sizes.
|
|
||||||
All the memory blocks are consecutive within a page, so the size of a block
|
|
||||||
also tells where the next block begins. Paired with an additional mechanism
|
|
||||||
(page type dependent) for determining whether a block is allocated, we can
|
|
||||||
iterate through the allocated blocks.
|
|
||||||
|
|
||||||
When a thread allocates memory for an object, it will find a page suitable for
|
|
||||||
the given allocation size. Each thread holds on to a number of pages for the
|
|
||||||
different size categories. The typical case is that the thread’s current page
|
|
||||||
for the given size can fit the requested allocation. If that is not the case, a
|
|
||||||
different page for that size category is requested from the shared allocation
|
|
||||||
space. The requested page can either be readily available (already prepared by
|
|
||||||
the GC thread), or it might need to be swept first, or it might be newly
|
|
||||||
created.
|
|
||||||
|
|
||||||
The GC thread has a new responsibility when using this allocator: While the
|
|
||||||
mutator threads are paused at the start of GC, the GC thread must prepare the
|
|
||||||
allocator for sweeping. This does two things. First, it marks all pages as
|
|
||||||
“needs to be swept before next use”. Second, it releases pages that threads are
|
|
||||||
holding on to, by clearing the thread local variables for each thread.
|
|
||||||
|
|
||||||
It is possible to have several independent allocation spaces at the same time.
|
|
||||||
This is currently useful for testing, but is potentially useful in other
|
|
||||||
settings.
|
|
||||||
|
|
||||||
# Detailed Design
|
|
||||||
|
|
||||||
All allocations are made through a `CustomAllocator` object.
|
|
||||||
|
|
||||||
## [CustomAllocator](cpp/CustomAllocator.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class CustomAllocator {
|
|
||||||
public:
|
|
||||||
CustomAllocator(Heap& heap, GCSchedulerThreadData& scheduler);
|
|
||||||
ObjectHeader* CreateObject(TypeInfo* type);
|
|
||||||
ArrayHeader* CreateArray(TypeInfo* type, uint32_t count);
|
|
||||||
ExtraObjectData* CreateExtraObject();
|
|
||||||
void PrepareForGc();
|
|
||||||
|
|
||||||
private:
|
|
||||||
uint8_t* Allocate(uint64_t cellCount);
|
|
||||||
uint8_t* AllocateInSingleObjectPage(uint64_t cellCount);
|
|
||||||
uint8_t* AllocateInNextFitPage(uint32_t cellCount);
|
|
||||||
uint8_t* AllocateInFixedBlockPage(uint32_t cellCount);
|
|
||||||
|
|
||||||
Heap& heap_;
|
|
||||||
GCSchedulerThreadData& gcScheduler_;
|
|
||||||
NextFitPage* nextFitPage_;
|
|
||||||
FixedBlockPage* fixedBlockPages_[MAX_BLOCK_SIZE];
|
|
||||||
ExtraObjectPage* extraObjectPage_;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
The primary responsibility of this class is to delegate each requested
|
|
||||||
allocation to pages of the appropriate type, based on allocation size. To do
|
|
||||||
this, it requests pages from the shared allocation space (`Heap`) and stores
|
|
||||||
pages for later allocations. Each thread thus owns a number of pages for
|
|
||||||
different allocation sizes, but at most one for each size class. When
|
|
||||||
allocating, the `CustomAllocator` will first try to allocate in one of its
|
|
||||||
owned pages. If this fails, it will request a new page for that size class from
|
|
||||||
a shared `Heap` object. `SingleObjectPages` are never kept by the
|
|
||||||
`CustomAllocator`, since they are created specifically for a single allocation,
|
|
||||||
with no extra space.
|
|
||||||
|
|
||||||
## [Heap](cpp/Heap.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class Heap {
|
|
||||||
public:
|
|
||||||
void PrepareForGC();
|
|
||||||
|
|
||||||
void Sweep();
|
|
||||||
|
|
||||||
AtomicStack<ExtraObjectCell> SweepExtraObjects(GCHandle gcHandle);
|
|
||||||
|
|
||||||
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount);
|
|
||||||
NextFitPage* GetNextFitPage(uint32_t cellCount);
|
|
||||||
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount);
|
|
||||||
ExtraObjectPage* GetExtraObjectPage();
|
|
||||||
|
|
||||||
private:
|
|
||||||
PageStore<FixedBlockPage> fixedBlockPages_[MAX_BLOCK_SIZE];
|
|
||||||
PageStore<NextFitPage> nextFitPages_;
|
|
||||||
PageStore<SingleObjectPage> singleObjectPages_;
|
|
||||||
AtomicStack<ExtraObjectPage> extraObjectPages_;
|
|
||||||
AtomicStack<ExtraObjectPage> usedExtraObjectPages_;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
A `Heap` object represents a shared allocation space for multiple
|
|
||||||
`CustomAllocator`s, which can request pages through one of the
|
|
||||||
`GetFixedBlockPage`, `GetNextFitPage`, `GetSingleObjectPage` methods. It also
|
|
||||||
provides a method for sweeping through all blocks that have been allocated in
|
|
||||||
this heap. The `Heap` object is the synchronization point, and guarantees that
|
|
||||||
every page is returned at most once. Page ownership is thus implicitly given to
|
|
||||||
the thread that called the method. The `Heap` object keeps track of all pages,
|
|
||||||
so there is no need to explicitly return ownership of a page. Internally, a
|
|
||||||
`Heap` keeps the pages for each size class in a `PageStore`. This means one for
|
|
||||||
`SingleObjectPage`s, one for `NextFitPage`s, one for each of the block sizes
|
|
||||||
for `FixedBlockPage`s. `ExtraObjectPage`s are stored directly in two
|
|
||||||
`AtomicStack`s, since they require different handling during sweeping.
|
|
||||||
|
|
||||||
## [PageStore](cpp/PageStore.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
template <class PageType>
|
|
||||||
class PageStore {
|
|
||||||
public:
|
|
||||||
void PrepareForGC();
|
|
||||||
void Sweep();
|
|
||||||
void SweepAndFree();
|
|
||||||
PageType* GetPage(uint32_t cellCount);
|
|
||||||
PageType* NewPage(uint64_t cellCount);
|
|
||||||
|
|
||||||
private:
|
|
||||||
AtomicStack<PageType> empty_;
|
|
||||||
AtomicStack<PageType> ready_;
|
|
||||||
AtomicStack<PageType> used_;
|
|
||||||
AtomicStack<PageType> unswept_;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
A PageStore is responsible for keeping track of all pages of a given type and
|
|
||||||
size class. Each of the pages are in one of four stacks. The stack, that a
|
|
||||||
given page is in, determines its current state:
|
|
||||||
|
|
||||||
* `unswept_`: have not yet been swept since the last GC cycle.
|
|
||||||
* `ready_`: are ready for allocation; has been swept by the GC thread.
|
|
||||||
* `used_`: has been given to some thread for allocation; it might still be used
|
|
||||||
for allocation, or it might have been discarded with not enough space left.
|
|
||||||
Will not be used until the next GC cycle.
|
|
||||||
* `empty_`: same as `ready_`, but does not contain any objects. Will be freed
|
|
||||||
before the next GC, if not needed before then.
|
|
||||||
|
|
||||||
When a page is requested, the page is taken from `ready_`, if there are any.
|
|
||||||
Otherwise, an `unswept_` page is taken and swept before returning. If there are
|
|
||||||
no unswept pages either, an empty page is taken, if there are any. Otherwise a
|
|
||||||
new page is created in the size category. All returned pages are moved to
|
|
||||||
`used_`. During the marking phase, all remaining pages in `empty_` are freed,
|
|
||||||
and all other pages are moved to `unswept_`. The GC thread will go through all
|
|
||||||
`PageStore`s and sweep the pages in `unswept_` and move them to `ready_`. If one
|
|
||||||
of the other threads sweeps a page from `unswept_`, it is moved directly to
|
|
||||||
`used_`, as it is claimed by the `CustomAllocator` that swept it.
|
|
||||||
|
|
||||||
`SingleObjectPage`s are treated slightly differently, because they are created
|
|
||||||
for one specific single allocation, and not reused when that allocation is
|
|
||||||
freed. A `SingleObjectPage` allocation goes directly to `NewPage(...)`, without
|
|
||||||
checking any of the stacks, and during sweeping, they are freed directly rather
|
|
||||||
than being put into the `empty_` stack. Ideally, there would only be two stacks
|
|
||||||
in play for `SingleObjectPages`; `used_` and `unswept_`. However, very little
|
|
||||||
is lost by just using the existing `PageStore` logic used for the other pages.
|
|
||||||
|
|
||||||
## [AtomicStack](cpp/AtomicStack.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
template <class PageType>
|
|
||||||
class AtomicStack {
|
|
||||||
public:
|
|
||||||
PageType* Pop();
|
|
||||||
void Push(PageType* elm);
|
|
||||||
void TransferAllFrom(AtomicStack<T>& src);
|
|
||||||
bool isEmpty();
|
|
||||||
|
|
||||||
|
|
||||||
private:
|
|
||||||
std::atomic<PageType*> stack_;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
The only place where atomics are used are in the stacks inside the `PageStore`.
|
|
||||||
All page classes have a non-atomic next pointer, to be used for linking up in
|
|
||||||
exactly one stack. `Pop` and `Push` are implemented with compare-and-swap
|
|
||||||
operations. The class is thread safe, except for if an element is freed while
|
|
||||||
another thread tries to Pop it from a stack.
|
|
||||||
|
|
||||||
# Page types
|
|
||||||
|
|
||||||
This section is likely to change, given the likely introduction of additional
|
|
||||||
page types. It also describes some details about which page type is chosen for
|
|
||||||
a given allocation, which is also likely to change.
|
|
||||||
|
|
||||||
There are four different page types, but they all share the feature that they
|
|
||||||
can be swept independently. The Sweep methods return whether there were any
|
|
||||||
live objects in the page after sweeping. If not, the page will be given back to
|
|
||||||
the OS.
|
|
||||||
|
|
||||||
## [FixedBlockPage](cpp/FixedBlockPage.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class FixedBlockPage {
|
|
||||||
public:
|
|
||||||
FixedBlockPage(uint32_t blockSize) noexcept;
|
|
||||||
|
|
||||||
uint8_t* TryAllocate() noexcept;
|
|
||||||
|
|
||||||
bool Sweep() noexcept;
|
|
||||||
|
|
||||||
private:
|
|
||||||
FixedBlockPage* next_;
|
|
||||||
FixedCellRange nextFree_;
|
|
||||||
uint32_t blockSize_;
|
|
||||||
uint32_t end_;
|
|
||||||
FixedBlockCell cells_[];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
All sufficiently small allocations (currently arbitrary <1KiB) are directed to
|
|
||||||
a `FixedBlockPage`, where all blocks have the same fixed size. Most allocations
|
|
||||||
are expected to be in this page type. A `FixedBlockPage` consists of a number
|
|
||||||
of equally sized blocks, where each allocation will take up exactly one such
|
|
||||||
block.
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
struct FixedBlockCell {
|
|
||||||
union {
|
|
||||||
uint8_t data[];
|
|
||||||
FixedCellRange nextFree;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct alignas(8) FixedCellRange {
|
|
||||||
uint32_t first;
|
|
||||||
uint32_t last;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
Consecutive unallocated cells are represented by a `FixedCellRange`, with
|
|
||||||
`.first` and `.last` being the inclusive end points of the range of unallocated
|
|
||||||
cells. The `FixedBlockCell` at the the `.last` index contains a
|
|
||||||
`FixedCellRange` with the next range of unallocated cells. The `FixedCellRange`
|
|
||||||
of unallocated ranges thus form a linked list.
|
|
||||||
|
|
||||||
The important point is that all links in this list point forward in the page, so
|
|
||||||
all blocks between two `FixedCellRanges` are implicitly allocated. Sweeping a
|
|
||||||
`FixedBlockPage` consists of walking the free-list forward, and sweeping all
|
|
||||||
blocks in between the links, maintaining the free list when blocks are freed.
|
|
||||||
|
|
||||||
Each small page takes up the same amount of space, independent of block size,
|
|
||||||
so larger block size implies fewer blocks per page. This size is arbitrarily
|
|
||||||
chosen to be 256 KiB, but this might change.
|
|
||||||
|
|
||||||
## [NextFitPage](cpp/NextFitPage.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class NextFitPage {
|
|
||||||
public:
|
|
||||||
NextFitPage(uint32_t cellCount);
|
|
||||||
Cell* TryAllocate(uint32_t cellCount);
|
|
||||||
bool Sweep();
|
|
||||||
|
|
||||||
private:
|
|
||||||
NextFitPage* next_; // used by AtomicStack
|
|
||||||
Cell* curBlock_;
|
|
||||||
Cell cells_[];
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
Allocations that could theoretically fit in a `FixedBlockPage`, but would
|
|
||||||
require too large a block size (arbitrary >=1KiB), are allocated in a
|
|
||||||
`NextFitPage`. `NextFitPage`s are the same size as `FixedBlockPage`s (arbitrary 64
|
|
||||||
KiB for experiments). All blocks in a `NextFitPage` have a header that tells how
|
|
||||||
big the block is, and whether it is allocated or not. There are no gaps between
|
|
||||||
blocks, so the size of a block also tells where the next block is. The header
|
|
||||||
information fits inside a 8 byte `Cell`.
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class Cell {
|
|
||||||
public:
|
|
||||||
Cell(uint32_t size);
|
|
||||||
uint8_t* TryAllocate(uint32_t cellCount);
|
|
||||||
|
|
||||||
private:
|
|
||||||
uint32_t isAllocated_;
|
|
||||||
uint32_t size_;
|
|
||||||
uint8_t data_[];
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
The page keeps a reference to a currently active block, and will try to bump
|
|
||||||
allocate inside that block. If allocation does not fit, we move to the next
|
|
||||||
block that fits. If no block in the page fits the requested size, the page is
|
|
||||||
abandoned until the next GC.
|
|
||||||
|
|
||||||
## [SingleObjectPage](cpp/SingleObjectPage.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class SingleObjectPage {
|
|
||||||
public:
|
|
||||||
SingleObjectPage(uint64_t cellCount);
|
|
||||||
bool Sweep();
|
|
||||||
|
|
||||||
private:
|
|
||||||
SingleObjectPage* next_; // used by AtomicStack
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
Allocations too big for a `NextFitPage` are allocated in a `SingleObjectPage`,
|
|
||||||
which only contains that single block of the requested size. They are also
|
|
||||||
handled slightly differently by both `Heap` and `CustomAllocator`. First off,
|
|
||||||
`Heap::GetSingleObjectPage` will never check existing pages, and instead just
|
|
||||||
allocate a new page. Secondly, a `CustomAllocator` does not keep a reference to
|
|
||||||
any of the `SingleObjectPage`s. As a consequence, they are only swept by the GC
|
|
||||||
thread.
|
|
||||||
|
|
||||||
## [ExtraObjectPage](cpp/ExtraObjectPage.hpp)
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
class ExtraObjectPage {
|
|
||||||
public:
|
|
||||||
ExtraObjectPage();
|
|
||||||
ExtraObjectData* TryAllocate();
|
|
||||||
bool Sweep(FinalizerQueue& queue);
|
|
||||||
|
|
||||||
|
|
||||||
private:
|
|
||||||
ExtraObjectPage* next_; // used by AtomicStack
|
|
||||||
ExtraObjectCell* nextFree_;
|
|
||||||
ExtraObjectCell cells_[];
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
Extra objects are used for attaching additional data to some objects. This is
|
|
||||||
used for objects that require special handling during garbage collection:
|
|
||||||
|
|
||||||
* objects with finalizers
|
|
||||||
* weak references
|
|
||||||
* interop references
|
|
||||||
|
|
||||||
Extra objects are allocated in `ExtraObjectPage`s, which are very similar to
|
|
||||||
`FixedBlockPage`s. They primarily differ in how they are swept, since it is
|
|
||||||
during sweeping of `ExtraObject`s that scheduling of finalization happens. If
|
|
||||||
an object that requires finalization is found, it is added to the
|
|
||||||
`FinalizerQueue` given as argument. The cells are also slightly different, in
|
|
||||||
that they add a new field that allows the cells to be added to the finalizer
|
|
||||||
queue.
|
|
||||||
|
|
||||||
```cpp
|
|
||||||
struct ExtraObjectCell {
|
|
||||||
ExtraObjectCell* next_; // used by AtomicStack
|
|
||||||
ExtraObjectData data_;
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
# Finalizers
|
|
||||||
|
|
||||||
Section like to change.
|
|
||||||
|
|
||||||
In the existing memory model, finalization tasks are found and scheduled during
|
|
||||||
sweeping of regular objects. The objects to be finalized are chained together
|
|
||||||
using a pointer in the Node header, added to all allocated objects. This header
|
|
||||||
is not needed in the custom allocator, apart from linking in the finalization
|
|
||||||
queue.
|
|
||||||
|
|
||||||
We therefore reintroduce this pointer in a header for `ExtraObjectData`. For
|
|
||||||
this, we reuse the `ExtraObjectCell` as header for both free list pointer and as
|
|
||||||
linking pointer for the `AtomicStack` that we use as the `FinalizerQueue`.
|
|
||||||
|
|
||||||
# Enabling the allocator
|
|
||||||
|
|
||||||
The custom allocator is enabled with the compiler flag -Xallocator=custom.
|
|
||||||
|
|||||||
@@ -1,33 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
|
||||||
* that can be found in the LICENSE file.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "CustomLogging.hpp"
|
|
||||||
|
|
||||||
#include "GCApi.hpp"
|
|
||||||
|
|
||||||
// These functions are just stubs to make the existing object creation
|
|
||||||
// infrastructure link correctly, but if they are ever called, something went
|
|
||||||
// wrong.
|
|
||||||
|
|
||||||
namespace kotlin {
|
|
||||||
|
|
||||||
void* allocateInObjectPool(size_t size) noexcept {
|
|
||||||
CustomAllocWarning("static allocateInObjectPool(%zu) not supported", size);
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void freeInObjectPool(void* ptr, size_t size) noexcept {
|
|
||||||
CustomAllocWarning("static freeInObjectPool(%p, %zu) not supported", ptr, size);
|
|
||||||
}
|
|
||||||
|
|
||||||
void initObjectPool() noexcept {}
|
|
||||||
void compactObjectPoolInMainThread() noexcept {}
|
|
||||||
void compactObjectPoolInCurrentThread() noexcept {}
|
|
||||||
|
|
||||||
size_t allocatedBytes() noexcept {
|
|
||||||
return alloc::GetAllocatedBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace kotlin
|
|
||||||
@@ -7,6 +7,8 @@
|
|||||||
|
|
||||||
#ifdef CUSTOM_ALLOCATOR
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
|
||||||
|
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "CustomAllocator.hpp"
|
#include "CustomAllocator.hpp"
|
||||||
#include "CustomFinalizerProcessor.hpp"
|
#include "CustomFinalizerProcessor.hpp"
|
||||||
#include "GCApi.hpp"
|
#include "GCApi.hpp"
|
||||||
@@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
|
|||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "ExtraObjectDataFactory.hpp"
|
#include "ExtraObjectDataFactory.hpp"
|
||||||
#include "GC.hpp"
|
#include "GC.hpp"
|
||||||
#include "GlobalData.hpp"
|
#include "GlobalData.hpp"
|
||||||
#include "Logging.hpp"
|
#include "Logging.hpp"
|
||||||
#include "ObjectFactory.hpp"
|
#include "ObjectFactory.hpp"
|
||||||
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
|
#include "ObjectFactorySweep.hpp"
|
||||||
|
|
||||||
namespace kotlin::gc {
|
namespace kotlin::gc {
|
||||||
|
|
||||||
struct ObjectFactoryTraits {
|
struct ObjectFactoryTraits {
|
||||||
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
|
using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
|
||||||
using ObjectData = gc::GC::ObjectData;
|
using ObjectData = gc::GC::ObjectData;
|
||||||
|
|
||||||
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
|
Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
|
||||||
|
|
||||||
void OnOOM(size_t size) noexcept {
|
void OnOOM(size_t size) noexcept {
|
||||||
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
||||||
@@ -51,7 +56,7 @@ struct ObjectFactoryTraits {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||||
|
|
||||||
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
||||||
return ObjectFactory::NodeRef::From(object).ObjectData();
|
return ObjectFactory::NodeRef::From(object).ObjectData();
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
|
|||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
|
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
|
||||||
ObjectFactory& objectFactory,
|
ObjectFactory& objectFactory,
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||||
gcScheduler::GCScheduler& gcScheduler,
|
gcScheduler::GCScheduler& gcScheduler,
|
||||||
bool mutatorsCooperate,
|
bool mutatorsCooperate,
|
||||||
std::size_t auxGCThreads) noexcept :
|
std::size_t auxGCThreads) noexcept :
|
||||||
@@ -211,11 +211,11 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
|||||||
resumeTheWorld(gcHandle);
|
resumeTheWorld(gcHandle);
|
||||||
|
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||||
extraObjectFactoryIterable = std::nullopt;
|
extraObjectFactoryIterable = std::nullopt;
|
||||||
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||||
objectFactoryIterable = std::nullopt;
|
objectFactoryIterable = std::nullopt;
|
||||||
kotlin::compactObjectPoolInMainThread();
|
alloc::compactObjectPoolInMainThread();
|
||||||
#else
|
#else
|
||||||
// also sweeps extraObjects
|
// also sweeps extraObjects
|
||||||
auto finalizerQueue = heap_.Sweep(gcHandle);
|
auto finalizerQueue = heap_.Sweep(gcHandle);
|
||||||
@@ -224,7 +224,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
|||||||
}
|
}
|
||||||
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
||||||
#endif
|
#endif
|
||||||
scheduler.onGCFinish(epoch, allocatedBytes());
|
scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
|
||||||
state_.finish(epoch);
|
state_.finish(epoch);
|
||||||
gcHandle.finalizersScheduled(finalizerQueue.size());
|
gcHandle.finalizersScheduled(finalizerQueue.size());
|
||||||
gcHandle.finished();
|
gcHandle.finished();
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ public:
|
|||||||
#else
|
#else
|
||||||
ConcurrentMarkAndSweep(
|
ConcurrentMarkAndSweep(
|
||||||
ObjectFactory& objectFactory,
|
ObjectFactory& objectFactory,
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||||
gcScheduler::GCScheduler& scheduler,
|
gcScheduler::GCScheduler& scheduler,
|
||||||
bool mutatorsCooperate,
|
bool mutatorsCooperate,
|
||||||
std::size_t auxGCThreads) noexcept;
|
std::size_t auxGCThreads) noexcept;
|
||||||
@@ -93,7 +93,7 @@ private:
|
|||||||
|
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory& objectFactory_;
|
ObjectFactory& objectFactory_;
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||||
#else
|
#else
|
||||||
alloc::Heap heap_;
|
alloc::Heap heap_;
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -9,7 +9,6 @@
|
|||||||
#include "GC.hpp"
|
#include "GC.hpp"
|
||||||
#include "GCStatistics.hpp"
|
#include "GCStatistics.hpp"
|
||||||
#include "MarkAndSweepUtils.hpp"
|
#include "MarkAndSweepUtils.hpp"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "ObjectOps.hpp"
|
#include "ObjectOps.hpp"
|
||||||
#include "std_support/Memory.hpp"
|
#include "std_support/Memory.hpp"
|
||||||
|
|
||||||
@@ -100,7 +99,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
return allocatedBytes();
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
return alloc::GetAllocatedBytes();
|
||||||
|
#else
|
||||||
|
return alloc::allocatedBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void gc::GC::ClearForTests() noexcept {
|
void gc::GC::ClearForTests() noexcept {
|
||||||
|
|||||||
@@ -23,14 +23,14 @@ public:
|
|||||||
|
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||||
#endif
|
#endif
|
||||||
ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
|
ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory objectFactory_;
|
ObjectFactory objectFactory_;
|
||||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||||
#endif
|
#endif
|
||||||
ConcurrentMarkAndSweep gc_;
|
ConcurrentMarkAndSweep gc_;
|
||||||
};
|
};
|
||||||
@@ -53,7 +53,7 @@ public:
|
|||||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||||
#else
|
#else
|
||||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@@ -63,7 +63,7 @@ private:
|
|||||||
#else
|
#else
|
||||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@
|
|||||||
#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
|
#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
|
||||||
|
|
||||||
#include "ExtraObjectData.hpp"
|
#include "ExtraObjectData.hpp"
|
||||||
#include "ExtraObjectDataFactory.hpp"
|
|
||||||
#include "FinalizerHooks.hpp"
|
#include "FinalizerHooks.hpp"
|
||||||
#include "GlobalData.hpp"
|
#include "GlobalData.hpp"
|
||||||
#include "GCStatistics.hpp"
|
#include "GCStatistics.hpp"
|
||||||
@@ -106,79 +105,6 @@ void Mark(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQue
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename Traits>
|
|
||||||
void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::Iterable& factoryIter) noexcept {
|
|
||||||
auto sweepHandle = handle.sweepExtraObjects();
|
|
||||||
factoryIter.ApplyDeletions();
|
|
||||||
for (auto it = factoryIter.begin(); it != factoryIter.end();) {
|
|
||||||
auto &extraObject = *it;
|
|
||||||
if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) {
|
|
||||||
extraObject.ClearRegularWeakReferenceImpl();
|
|
||||||
if (extraObject.HasAssociatedObject()) {
|
|
||||||
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
|
|
||||||
++it;
|
|
||||||
sweepHandle.addKeptObject();
|
|
||||||
} else {
|
|
||||||
extraObject.Uninstall();
|
|
||||||
it.EraseAndAdvance();
|
|
||||||
sweepHandle.addSweptObject();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
++it;
|
|
||||||
sweepHandle.addKeptObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename Traits>
|
|
||||||
void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& factory) noexcept {
|
|
||||||
auto iter = factory.LockForIter();
|
|
||||||
return SweepExtraObjects<Traits>(handle, iter);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename Traits>
|
|
||||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
|
|
||||||
typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
|
|
||||||
auto sweepHandle = handle.sweep();
|
|
||||||
|
|
||||||
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
|
|
||||||
if (Traits::TryResetMark(*it)) {
|
|
||||||
++it;
|
|
||||||
sweepHandle.addKeptObject();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
sweepHandle.addSweptObject();
|
|
||||||
auto* objHeader = it->GetObjHeader();
|
|
||||||
if (HasFinalizers(objHeader)) {
|
|
||||||
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
|
|
||||||
} else {
|
|
||||||
objectFactoryIter.EraseAndAdvance(it);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return finalizerQueue;
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename Traits>
|
|
||||||
typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
|
|
||||||
auto iter = objectFactory.LockForIter();
|
|
||||||
return Sweep<Traits>(handle, iter);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename T>
|
|
||||||
struct DefaultSweepTraits {
|
|
||||||
using ObjectFactory = T;
|
|
||||||
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
|
|
||||||
|
|
||||||
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
|
|
||||||
auto* baseObject = object.GetBaseObject();
|
|
||||||
if (!baseObject->heap()) return true;
|
|
||||||
return gc::isMarked(baseObject);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
|
|
||||||
};
|
|
||||||
|
|
||||||
template <typename Traits>
|
template <typename Traits>
|
||||||
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
|
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
|
||||||
auto handle = gcHandle.collectThreadRoots(thread);
|
auto handle = gcHandle.collectThreadRoots(thread);
|
||||||
|
|||||||
@@ -7,26 +7,31 @@
|
|||||||
|
|
||||||
#ifdef CUSTOM_ALLOCATOR
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
|
||||||
|
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "CustomAllocator.hpp"
|
#include "CustomAllocator.hpp"
|
||||||
|
#include "GCApi.hpp"
|
||||||
#include "Heap.hpp"
|
#include "Heap.hpp"
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "ExtraObjectDataFactory.hpp"
|
#include "ExtraObjectDataFactory.hpp"
|
||||||
#include "GC.hpp"
|
#include "GC.hpp"
|
||||||
#include "ObjectFactory.hpp"
|
#include "ObjectFactory.hpp"
|
||||||
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
|
|
||||||
namespace kotlin::gc {
|
namespace kotlin::gc {
|
||||||
|
|
||||||
struct ObjectFactoryTraits {
|
struct ObjectFactoryTraits {
|
||||||
using Allocator = Allocator;
|
using Allocator = alloc::AllocatorBasic;
|
||||||
using ObjectData = gc::GC::ObjectData;
|
using ObjectData = gc::GC::ObjectData;
|
||||||
|
|
||||||
Allocator CreateAllocator() noexcept { return Allocator(); }
|
Allocator CreateAllocator() noexcept { return Allocator(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||||
|
|
||||||
} // namespace kotlin::gc
|
} // namespace kotlin::gc
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@
|
|||||||
#include "GC.hpp"
|
#include "GC.hpp"
|
||||||
#include "GCStatistics.hpp"
|
#include "GCStatistics.hpp"
|
||||||
#include "NoOpGC.hpp"
|
#include "NoOpGC.hpp"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "ObjectOps.hpp"
|
#include "ObjectOps.hpp"
|
||||||
#include "ThreadData.hpp"
|
#include "ThreadData.hpp"
|
||||||
#include "std_support/Memory.hpp"
|
#include "std_support/Memory.hpp"
|
||||||
@@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
return allocatedBytes();
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
return alloc::GetAllocatedBytes();
|
||||||
|
#else
|
||||||
|
return alloc::allocatedBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void gc::GC::ClearForTests() noexcept {
|
void gc::GC::ClearForTests() noexcept {
|
||||||
|
|||||||
@@ -19,14 +19,14 @@ public:
|
|||||||
|
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||||
#endif
|
#endif
|
||||||
NoOpGC& gc() noexcept { return gc_; }
|
NoOpGC& gc() noexcept { return gc_; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory objectFactory_;
|
ObjectFactory objectFactory_;
|
||||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||||
#endif
|
#endif
|
||||||
NoOpGC gc_;
|
NoOpGC gc_;
|
||||||
};
|
};
|
||||||
@@ -46,7 +46,7 @@ public:
|
|||||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||||
#else
|
#else
|
||||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@@ -56,7 +56,7 @@ private:
|
|||||||
#else
|
#else
|
||||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,8 @@
|
|||||||
|
|
||||||
#ifdef CUSTOM_ALLOCATOR
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
|
||||||
|
// TODO: Move into alloc/custom/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "CustomAllocator.hpp"
|
#include "CustomAllocator.hpp"
|
||||||
#include "CustomFinalizerProcessor.hpp"
|
#include "CustomFinalizerProcessor.hpp"
|
||||||
#include "GCApi.hpp"
|
#include "GCApi.hpp"
|
||||||
@@ -29,20 +31,23 @@ using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
|
|||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
#include "Allocator.hpp"
|
// TODO: Move into alloc/legacy/AllocatorImpl.hpp
|
||||||
|
|
||||||
#include "ExtraObjectDataFactory.hpp"
|
#include "ExtraObjectDataFactory.hpp"
|
||||||
#include "GC.hpp"
|
#include "GC.hpp"
|
||||||
#include "GlobalData.hpp"
|
#include "GlobalData.hpp"
|
||||||
#include "Logging.hpp"
|
#include "Logging.hpp"
|
||||||
#include "ObjectFactory.hpp"
|
#include "ObjectFactory.hpp"
|
||||||
|
#include "ObjectFactoryAllocator.hpp"
|
||||||
|
#include "ObjectFactorySweep.hpp"
|
||||||
|
|
||||||
namespace kotlin::gc {
|
namespace kotlin::gc {
|
||||||
|
|
||||||
struct ObjectFactoryTraits {
|
struct ObjectFactoryTraits {
|
||||||
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
|
using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
|
||||||
using ObjectData = gc::GC::ObjectData;
|
using ObjectData = gc::GC::ObjectData;
|
||||||
|
|
||||||
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
|
Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
|
||||||
|
|
||||||
void OnOOM(size_t size) noexcept {
|
void OnOOM(size_t size) noexcept {
|
||||||
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
|
||||||
@@ -51,7 +56,7 @@ struct ObjectFactoryTraits {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
|
using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
|
||||||
|
|
||||||
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
|
||||||
return ObjectFactory::NodeRef::From(object).ObjectData();
|
return ObjectFactory::NodeRef::From(object).ObjectData();
|
||||||
|
|||||||
@@ -9,7 +9,6 @@
|
|||||||
#include "GCStatistics.hpp"
|
#include "GCStatistics.hpp"
|
||||||
#include "GlobalData.hpp"
|
#include "GlobalData.hpp"
|
||||||
#include "MarkAndSweepUtils.hpp"
|
#include "MarkAndSweepUtils.hpp"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "ObjectOps.hpp"
|
#include "ObjectOps.hpp"
|
||||||
#include "SameThreadMarkAndSweep.hpp"
|
#include "SameThreadMarkAndSweep.hpp"
|
||||||
#include "std_support/Memory.hpp"
|
#include "std_support/Memory.hpp"
|
||||||
@@ -89,7 +88,11 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
return allocatedBytes();
|
#ifdef CUSTOM_ALLOCATOR
|
||||||
|
return alloc::GetAllocatedBytes();
|
||||||
|
#else
|
||||||
|
return alloc::allocatedBytes();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void gc::GC::ClearForTests() noexcept {
|
void gc::GC::ClearForTests() noexcept {
|
||||||
|
|||||||
@@ -21,14 +21,14 @@ public:
|
|||||||
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
|
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
|
||||||
|
|
||||||
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
|
||||||
#endif
|
#endif
|
||||||
SameThreadMarkAndSweep& gc() noexcept { return gc_; }
|
SameThreadMarkAndSweep& gc() noexcept { return gc_; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory objectFactory_;
|
ObjectFactory objectFactory_;
|
||||||
mm::ExtraObjectDataFactory extraObjectDataFactory_;
|
alloc::ExtraObjectDataFactory extraObjectDataFactory_;
|
||||||
#endif
|
#endif
|
||||||
SameThreadMarkAndSweep gc_;
|
SameThreadMarkAndSweep gc_;
|
||||||
};
|
};
|
||||||
@@ -49,7 +49,7 @@ public:
|
|||||||
SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
|
SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
|
||||||
#else
|
#else
|
||||||
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
|
||||||
#endif
|
#endif
|
||||||
@@ -61,7 +61,7 @@ private:
|
|||||||
#else
|
#else
|
||||||
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
|
||||||
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
|
||||||
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
#include "Logging.hpp"
|
#include "Logging.hpp"
|
||||||
#include "MarkAndSweepUtils.hpp"
|
#include "MarkAndSweepUtils.hpp"
|
||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "RootSet.hpp"
|
#include "RootSet.hpp"
|
||||||
#include "Runtime.h"
|
#include "Runtime.h"
|
||||||
#include "ThreadData.hpp"
|
#include "ThreadData.hpp"
|
||||||
@@ -28,7 +27,7 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
|
|||||||
gcScheduler::GCScheduler& gcScheduler) noexcept :
|
gcScheduler::GCScheduler& gcScheduler) noexcept :
|
||||||
#else
|
#else
|
||||||
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
|
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
|
||||||
ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
|
ObjectFactory& objectFactory, alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
|
||||||
|
|
||||||
objectFactory_(objectFactory),
|
objectFactory_(objectFactory),
|
||||||
extraObjectDataFactory_(extraObjectDataFactory),
|
extraObjectDataFactory_(extraObjectDataFactory),
|
||||||
@@ -103,11 +102,11 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
|||||||
std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
|
std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
|
||||||
std::optional objectFactoryIterable = objectFactory_.LockForIter();
|
std::optional objectFactoryIterable = objectFactory_.LockForIter();
|
||||||
|
|
||||||
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
|
||||||
extraObjectFactoryIterable = std::nullopt;
|
extraObjectFactoryIterable = std::nullopt;
|
||||||
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
|
||||||
objectFactoryIterable = std::nullopt;
|
objectFactoryIterable = std::nullopt;
|
||||||
kotlin::compactObjectPoolInMainThread();
|
alloc::compactObjectPoolInMainThread();
|
||||||
#else
|
#else
|
||||||
// also sweeps extraObjects
|
// also sweeps extraObjects
|
||||||
auto finalizerQueue = heap_.Sweep(gcHandle);
|
auto finalizerQueue = heap_.Sweep(gcHandle);
|
||||||
@@ -117,7 +116,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
|
|||||||
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
scheduler.onGCFinish(epoch, allocatedBytes());
|
scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
|
||||||
|
|
||||||
resumeTheWorld(gcHandle);
|
resumeTheWorld(gcHandle);
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ public:
|
|||||||
#else
|
#else
|
||||||
SameThreadMarkAndSweep(
|
SameThreadMarkAndSweep(
|
||||||
ObjectFactory& objectFactory,
|
ObjectFactory& objectFactory,
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory,
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory,
|
||||||
gcScheduler::GCScheduler& gcScheduler) noexcept;
|
gcScheduler::GCScheduler& gcScheduler) noexcept;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@@ -66,7 +66,7 @@ private:
|
|||||||
|
|
||||||
#ifndef CUSTOM_ALLOCATOR
|
#ifndef CUSTOM_ALLOCATOR
|
||||||
ObjectFactory& objectFactory_;
|
ObjectFactory& objectFactory_;
|
||||||
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
|
alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
|
||||||
#else
|
#else
|
||||||
alloc::Heap heap_;
|
alloc::Heap heap_;
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -612,6 +612,9 @@ bool FinalizersThreadIsRunning() noexcept;
|
|||||||
|
|
||||||
void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept;
|
void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept;
|
||||||
|
|
||||||
|
void initObjectPool() noexcept;
|
||||||
|
void compactObjectPoolInCurrentThread() noexcept;
|
||||||
|
|
||||||
} // namespace kotlin
|
} // namespace kotlin
|
||||||
|
|
||||||
RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object);
|
RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object);
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
#include "MainQueueProcessor.hpp"
|
#include "MainQueueProcessor.hpp"
|
||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
#include "ObjCExportInit.h"
|
#include "ObjCExportInit.h"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "Porting.h"
|
#include "Porting.h"
|
||||||
#include "Runtime.h"
|
#include "Runtime.h"
|
||||||
#include "RuntimePrivate.hpp"
|
#include "RuntimePrivate.hpp"
|
||||||
|
|||||||
@@ -32,7 +32,6 @@
|
|||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
#include "Natives.h"
|
#include "Natives.h"
|
||||||
#include "ObjCMMAPI.h"
|
#include "ObjCMMAPI.h"
|
||||||
#include "ObjectAlloc.hpp"
|
|
||||||
#include "Runtime.h"
|
#include "Runtime.h"
|
||||||
#include "Types.h"
|
#include "Types.h"
|
||||||
#include "Worker.h"
|
#include "Worker.h"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
#include "Memory.h"
|
#include "Memory.h"
|
||||||
#include "MemoryPrivate.hpp"
|
#include "MemoryPrivate.hpp"
|
||||||
|
|
||||||
|
#include "Allocator.hpp"
|
||||||
#include "Exceptions.h"
|
#include "Exceptions.h"
|
||||||
#include "ExtraObjectData.hpp"
|
#include "ExtraObjectData.hpp"
|
||||||
#include "Freezing.hpp"
|
#include "Freezing.hpp"
|
||||||
@@ -636,3 +637,11 @@ RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakR
|
|||||||
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
|
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
|
||||||
mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
|
mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void kotlin::initObjectPool() noexcept {
|
||||||
|
alloc::initObjectPool();
|
||||||
|
}
|
||||||
|
|
||||||
|
void kotlin::compactObjectPoolInCurrentThread() noexcept {
|
||||||
|
alloc::compactObjectPoolInCurrentThread();
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user