From 9ebba93dd9c0949fc7eaf3453bf92eb293ea26bc Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Fri, 21 May 2021 13:59:22 +0300 Subject: [PATCH] Make SingleThreadMarkAndSweep support multiple threads --- .../backend.native/tests/build.gradle | 125 ++---- .../tests/objcexport/values.swift | 15 +- .../runtime/basic/cleaner_in_tls_worker.kt | 1 + .../tests/runtime/workers/freeze2.kt | 19 +- .../kotlin/bitcode/CompileToBitcode.kt | 2 +- .../runtime/src/gc/noop/cpp/NoOpGC.hpp | 7 +- kotlin-native/runtime/src/gc/stms/cpp/GC.hpp | 2 +- .../gc/stms/cpp/SingleThreadMarkAndSweep.cpp | 69 ++- .../gc/stms/cpp/SingleThreadMarkAndSweep.hpp | 18 +- .../stms/cpp/SingleThreadMarkAndSweepTest.cpp | 411 ++++++++++++++++++ .../runtime/src/mm/cpp/ExtraObjectData.cpp | 5 +- .../runtime/src/mm/cpp/ObjectFactory.hpp | 2 + .../runtime/src/mm/cpp/ThreadData.hpp | 2 +- .../runtime/src/mm/cpp/ThreadState.cpp | 2 +- .../test_support/cpp/CompilerGenerated.cpp | 3 +- 15 files changed, 556 insertions(+), 127 deletions(-) diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index 26186dbeedd..206a879a9dc 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -708,7 +708,7 @@ task runtime_basic_simd(type: KonanLocalTest) { task runtime_worker_random(type: KonanLocalTest) { enabled = (project.testTarget != 'wasm32') && // Uses workers. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. source = "runtime/basic/worker_random.kt" } @@ -826,170 +826,150 @@ task empty_substring(type: KonanLocalTest) { } standaloneTest("cleaner_basic") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_basic.kt" flags = ['-tr', '-Xopt-in=kotlin.native.internal.InternalForKotlinNative'] } standaloneTest("cleaner_workers") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_workers.kt" flags = ['-tr', '-Xopt-in=kotlin.native.internal.InternalForKotlinNative'] } standaloneTest("cleaner_in_main_with_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_in_main_with_checker.kt" goldValue = "42\n" } standaloneTest("cleaner_in_main_without_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_in_main_without_checker.kt" goldValue = "" } standaloneTest("cleaner_leak_without_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_leak_without_checker.kt" goldValue = "" } standaloneTest("cleaner_leak_with_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_leak_with_checker.kt" expectedExitStatusChecker = { it != 0 } outputChecker = { s -> (s =~ /Cleaner (0x)?[0-9a-fA-F]+ was disposed during program exit/).find() } } standaloneTest("cleaner_in_tls_main_without_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_in_tls_main_without_checker.kt" } standaloneTest("cleaner_in_tls_main_with_checker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_in_tls_main_with_checker.kt" expectedExitStatusChecker = { it != 0 } outputChecker = { s -> (s =~ /Cleaner (0x)?[0-9a-fA-F]+ was disposed during program exit/).find() } } standaloneTest("cleaner_in_tls_worker") { - enabled = (project.testTarget != 'wasm32') && // Cleaners need workers - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Cleaners need workers source = "runtime/basic/cleaner_in_tls_worker.kt" flags = ['-Xopt-in=kotlin.native.internal.InternalForKotlinNative'] } standaloneTest("worker_bound_reference0") { enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. source = "runtime/concurrent/worker_bound_reference0.kt" flags = ['-tr'] } task worker0(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "Got Input processed\nOK\n" source = "runtime/workers/worker0.kt" } task worker1(type: KonanLocalTest) { enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. goldValue = "OK\n" source = "runtime/workers/worker1.kt" } task worker2(type: KonanLocalTest) { enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. goldValue = "OK\n" source = "runtime/workers/worker2.kt" } task worker3(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "OK\n" source = "runtime/workers/worker3.kt" } task worker4(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "Got 42\nOK\n" source = "runtime/workers/worker4.kt" } // This tests changes main thread worker queue state, so better be executed alone. standaloneTest("worker5") { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "Got 3\nOK\n" source = "runtime/workers/worker5.kt" } task worker6(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "Got 42\nOK\n" source = "runtime/workers/worker6.kt" } task worker7(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "Input\nGot kotlin.Unit\nOK\n" source = "runtime/workers/worker7.kt" } task worker8(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "SharedData(string=Hello, int=10, member=SharedDataMember(double=0.1))\nGot kotlin.Unit\nOK\n" source = "runtime/workers/worker8.kt" } task worker9(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "zzz\n42\nOK\nfirst 2\nsecond 3\nfrozen OK\n" source = "runtime/workers/worker9.kt" } task worker10(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "OK\ntrue\ntrue\n" source = "runtime/workers/worker10.kt" } task worker11(type: KonanLocalTest) { enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. goldValue = "OK\n" source = "runtime/workers/worker11.kt" } standaloneTest("worker_threadlocal_no_leak") { - disabled = (project.testTarget == 'wasm32') || // Needs pthreads. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // Needs pthreads. source = "runtime/workers/worker_threadlocal_no_leak.kt" } task freeze0(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // No workers on WASM. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // No workers on WASM. goldValue = "frozen bit is true\n" + "Worker: SharedData(string=Hello, int=10, member=SharedDataMember(double=0.1))\n" + "Main: SharedData(string=Hello, int=10, member=SharedDataMember(double=0.1))\n" + @@ -1010,8 +990,7 @@ task freeze_stress(type: KonanLocalTest) { } task freeze2(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // No exceptions on WASM. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // No exceptions on WASM. goldValue = "Worker 1: Hello world\n" + "Worker2: 42\n" + "Worker3: 239.0\n" + "Worker4: a\n" + "OK\n" @@ -1019,8 +998,7 @@ task freeze2(type: KonanLocalTest) { } task freeze3(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // No exceptions on WASM. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // No exceptions on WASM. goldValue = "OK\n" source = "runtime/workers/freeze3.kt" } @@ -1044,7 +1022,7 @@ task freeze6(type: KonanLocalTest) { task atomic0(type: KonanLocalTest) { enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + !(isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. goldValue = "35\n" + "20\n" + "OK\n" source = "runtime/workers/atomic0.kt" } @@ -1056,8 +1034,7 @@ standaloneTest("atomic1") { } task lazy0(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "OK\n" source = "runtime/workers/lazy0.kt" } @@ -1078,29 +1055,25 @@ standaloneTest("lazy3") { } task mutableData1(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. Need exceptions - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. Need exceptions source = "runtime/workers/mutableData1.kt" } task enumIdentity(type: KonanLocalTest) { - enabled = (project.testTarget != 'wasm32') && // Workers need pthreads. - !isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + enabled = (project.testTarget != 'wasm32') // Workers need pthreads. goldValue = "true\n" source = "runtime/workers/enum_identity.kt" } standaloneTest("leakWorker") { - disabled = (project.testTarget == 'wasm32') || // Needs pthreads. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // Needs pthreads. source = "runtime/workers/leak_worker.kt" expectedExitStatusChecker = { it != 0 } outputChecker = { s -> s.contains("Unfinished workers detected, 1 workers leaked!") } } standaloneTest("leakMemoryWithWorkerTermination") { - disabled = (project.testTarget == 'wasm32') || // Needs pthreads. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // Needs pthreads. source = "runtime/workers/leak_memory_with_worker_termination.kt" if (!isExperimentalMM) { // Experimental MM will not report memory leaks. @@ -2841,7 +2814,7 @@ task initializers5(type: KonanLocalTest) { task initializers6(type: KonanLocalTest) { disabled = (project.testTarget == 'wasm32') || // Needs workers. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + (isExperimentalMM && project.testTarget == 'mingw_x64') // TODO: Fix Experimental MM performance on Windows. source = "runtime/basic/initializers6.kt" } @@ -2939,8 +2912,7 @@ standaloneTest("memory_only_gc") { } task memory_stable_ref_cross_thread_check(type: KonanLocalTest) { - disabled = (project.testTarget == 'wasm32') || // Needs workers. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // Needs workers. source = "runtime/memory/stable_ref_cross_thread_check.kt" } @@ -4224,15 +4196,13 @@ dynamicTest("interop_kt43502") { } interopTest("interop_leakMemoryWithRunningThreadUnchecked") { - disabled = (project.testTarget == 'wasm32') || // No interop for wasm yet. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // No interop for wasm yet. interop = 'leakMemoryWithRunningThread' source = "interop/leakMemoryWithRunningThread/unchecked.kt" } interopTest("interop_leakMemoryWithRunningThreadChecked") { - disabled = (project.testTarget == 'wasm32') || // No interop for wasm yet. - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.testTarget == 'wasm32') // No interop for wasm yet. interop = 'leakMemoryWithRunningThread' source = "interop/leakMemoryWithRunningThread/checked.kt" expectedExitStatusChecker = { it != 0 } @@ -4452,7 +4422,7 @@ if (PlatformInfo.isAppleTarget(project)) { } interopTest("interop_objc_illegal_sharing_with_weak") { - enabled = !isExperimentalMM // Experimental MM doesn't support multiple mutators and thread state switching for ObjC interop yet. + enabled = !isExperimentalMM // Experimental MM doesn't support thread state switching for ObjC interop yet. source = "interop/objc/illegal_sharing_with_weak/main.kt" interop = 'objc_illegal_sharing_with_weak' @@ -4465,7 +4435,7 @@ if (PlatformInfo.isAppleTarget(project)) { } interopTest("interop_objc_kt42172") { - enabled = !isExperimentalMM // Experimental MM doesn't support multiple mutators and thread state switching for ObjC interop yet. + enabled = !isExperimentalMM // Experimental MM doesn't support thread state switching for ObjC interop yet. source = "interop/objc/kt42172/main.kt" interop = "objc_kt42172" flags = ['-Xopt-in=kotlin.native.internal.InternalForKotlinNative'] @@ -4516,7 +4486,7 @@ standaloneTest("interop_zlib") { standaloneTest("interop_objc_illegal_sharing") { disabled = !isAppleTarget(project) || - isExperimentalMM // Experimental MM doesn't support multiple mutators and thread state switching for ObjC interop yet. + isExperimentalMM // Experimental MM doesn't support thread state switching for ObjC interop yet. source = "interop/objc/illegal_sharing.kt" UtilsKt.dependsOnPlatformLibs(it) if (isExperimentalMM) { @@ -4601,8 +4571,7 @@ dynamicTest("interop_kt42397") { } dynamicTest("interop_cleaners_main_thread") { - disabled = (project.target.name != project.hostName) || - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.target.name != project.hostName) source = "interop/cleaners/cleaners.kt" cSource = "$projectDir/interop/cleaners/main_thread.cpp" clangTool = "clang++" @@ -4611,8 +4580,7 @@ dynamicTest("interop_cleaners_main_thread") { } dynamicTest("interop_cleaners_second_thread") { - disabled = (project.target.name != project.hostName) || - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.target.name != project.hostName) source = "interop/cleaners/cleaners.kt" cSource = "$projectDir/interop/cleaners/second_thread.cpp" clangTool = "clang++" @@ -4621,8 +4589,7 @@ dynamicTest("interop_cleaners_second_thread") { } dynamicTest("interop_cleaners_leak") { - disabled = (project.target.name != project.hostName) || - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.target.name != project.hostName) source = "interop/cleaners/cleaners.kt" cSource = "$projectDir/interop/cleaners/leak.cpp" clangTool = "clang++" @@ -4641,8 +4608,7 @@ dynamicTest("interop_migrating_main_thread_legacy") { } dynamicTest("interop_migrating_main_thread") { - disabled = (project.target.name != project.hostName) || - isExperimentalMM // Experimental MM doesn't support multiple mutators yet. + disabled = (project.target.name != project.hostName) source = "interop/migrating_main_thread/lib.kt" flags = ['-Xdestroy-runtime-mode=on-shutdown'] if (isExperimentalMM) { @@ -5218,8 +5184,7 @@ private void configureStdlibTest(KonanGTest task, boolean inWorker) { task.useFilter = false task.testLogger = KonanTest.Logger.GTEST task.finalizedBy("resultsTask") - task.enabled = (project.testTarget != 'wasm32') && // Uses exceptions - (!inWorker || !ext.isExperimentalMM) // Experimental MM doesn't support multiple mutators yet. + task.enabled = (project.testTarget != 'wasm32') // Uses exceptions } /** diff --git a/kotlin-native/backend.native/tests/objcexport/values.swift b/kotlin-native/backend.native/tests/objcexport/values.swift index e0a2309dc74..297611283b1 100644 --- a/kotlin-native/backend.native/tests/objcexport/values.swift +++ b/kotlin-native/backend.native/tests/objcexport/values.swift @@ -1073,6 +1073,11 @@ class TestSharedRefs { } } + // This will free `object1` and release+dealloc its associated `Deinit` which nils `Deinit.object2` + ValuesKt.gc() + // This will free `object2`. + ValuesKt.gc() + try assertTrue(Deinit.weakVar2 === nil) } @@ -1403,10 +1408,7 @@ class ValuesTests : SimpleTestProvider { test("TestInvalidIdentifiers", testInvalidIdentifiers) test("TestDeprecation", testDeprecation) test("TestWeakRefs", testWeakRefs) - if !ValuesKt.isExperimentalMM { - // Experimental MM doesn't support multiple mutators yet. - test("TestSharedRefs", TestSharedRefs().test) - } + test("TestSharedRefs", TestSharedRefs().test) test("TestClassTypeCheck", testClassTypeCheck) test("TestInterfaceTypeCheck", testInterfaceTypeCheck) test("TestGH3503_1", testGH3503_1) @@ -1419,9 +1421,6 @@ class ValuesTests : SimpleTestProvider { test("TestFakeOverrideInInterface", testFakeOverrideInInterface) // Stress test, must remain the last one: - if !ValuesKt.isExperimentalMM { - // Experimental MM doesn't support multiple mutators yet. - test("TestGH2931", testGH2931) - } + test("TestGH2931", testGH2931) } } diff --git a/kotlin-native/backend.native/tests/runtime/basic/cleaner_in_tls_worker.kt b/kotlin-native/backend.native/tests/runtime/basic/cleaner_in_tls_worker.kt index e6ef93faeba..f43ee07f148 100644 --- a/kotlin-native/backend.native/tests/runtime/basic/cleaner_in_tls_worker.kt +++ b/kotlin-native/backend.native/tests/runtime/basic/cleaner_in_tls_worker.kt @@ -25,6 +25,7 @@ fun main() { worker.requestTermination().result waitWorkerTermination(worker) + GC.collect() waitCleanerWorker() assertEquals(42, value.value) diff --git a/kotlin-native/backend.native/tests/runtime/workers/freeze2.kt b/kotlin-native/backend.native/tests/runtime/workers/freeze2.kt index c0ede937cf9..0bb2be40ef4 100644 --- a/kotlin-native/backend.native/tests/runtime/workers/freeze2.kt +++ b/kotlin-native/backend.native/tests/runtime/workers/freeze2.kt @@ -59,27 +59,36 @@ data class Data(var int: Int) assertFailsWith { a8[1] = 2.0 } // Ensure that String and integral boxes are frozen by default, by passing local to the worker. + val hasToBeFrozen = Platform.memoryModel == MemoryModel.STRICT val worker = Worker.start() var data: Any = "Hello" + " " + "world" - assert(data.isFrozen) + if (hasToBeFrozen) { + assertTrue(data.isFrozen) + } worker.execute(TransferMode.SAFE, { data } ) { input -> println("Worker 1: $input") }.result data = 42 - assert(data.isFrozen) + if (hasToBeFrozen) { + assertTrue(data.isFrozen) + } worker.execute(TransferMode.SAFE, { data } ) { input -> println("Worker2: $input") }.result data = 239.0 - assert(data.isFrozen) + if (hasToBeFrozen) { + assertTrue(data.isFrozen) + } worker.execute(TransferMode.SAFE, { data } ) { input -> println("Worker3: $input") }.result data = 'a' - assert(data.isFrozen) + if (hasToBeFrozen) { + assertTrue(data.isFrozen) + } worker.execute(TransferMode.SAFE, { data } ) { input -> println("Worker4: $input") }.result @@ -87,4 +96,4 @@ data class Data(var int: Int) worker.requestTermination().result println("OK") -} \ No newline at end of file +} diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt index 4867b4e1d9e..d1c3feb11b0 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt @@ -44,7 +44,7 @@ open class CompileToBitcode @Inject constructor( // Source files and headers are registered as inputs by the `inputFiles` and `headers` properties. var srcDirs: FileCollection = project.files(srcRoot.resolve("cpp")) - var headersDirs: FileCollection = project.files(srcRoot.resolve("headers")) + var headersDirs: FileCollection = srcDirs + project.files(srcRoot.resolve("headers")) @Input var language = Language.CPP diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index b7ac98b46d9..1fab6ce305a 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -11,6 +11,11 @@ #include "Utils.hpp" namespace kotlin { + +namespace mm { +class ThreadData; +} + namespace gc { // No-op GC is a GC that does not free memory. @@ -23,7 +28,7 @@ public: public: using ObjectData = NoOpGC::ObjectData; - explicit ThreadData(NoOpGC& gc) noexcept {} + explicit ThreadData(NoOpGC& gc, mm::ThreadData& threadData) noexcept {} ~ThreadData() = default; void SafePointFunctionEpilogue() noexcept {} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp index a2d8ed45fd3..6d043d44ac2 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp @@ -13,7 +13,7 @@ namespace gc { using GC = kotlin::gc::SingleThreadMarkAndSweep; -inline constexpr bool kSupportsMultipleMutators = false; +inline constexpr bool kSupportsMultipleMutators = true; } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp index 11b920297a1..477281c5063 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp @@ -12,6 +12,7 @@ #include "Runtime.h" #include "ThreadData.hpp" #include "ThreadRegistry.hpp" +#include "ThreadSuspension.hpp" using namespace kotlin; @@ -49,49 +50,76 @@ struct FinalizeTraits { } // namespace void gc::SingleThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept { - if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) { - PerformFullGC(); - } - ++safePointsCounter_; + SafePointRegular(1); } void gc::SingleThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { - if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) { - PerformFullGC(); - } - ++safePointsCounter_; + SafePointRegular(1); } void gc::SingleThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept { - if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) { - PerformFullGC(); - } - ++safePointsCounter_; + SafePointRegular(1); } void gc::SingleThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { size_t allocationOverhead = gc_.GetAllocationThresholdBytes() == 0 ? allocatedBytes_ : allocatedBytes_ % gc_.GetAllocationThresholdBytes(); - if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) { + if (threadData_.suspensionData().suspendIfRequested()) { + allocatedBytes_ = 0; + } else if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) { + allocatedBytes_ = 0; PerformFullGC(); } allocatedBytes_ += size; } void gc::SingleThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept { - gc_.PerformFullGC(); + mm::ObjectFactory::FinalizerQueue finalizerQueue; + { + // Switch state to native to simulate this thread being a GC thread. + // As a bonus, if we failed to suspend threads (which means some other thread asked for a GC), + // we will automatically suspend at the scope exit. + // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`. + ThreadStateGuard guard(ThreadState::kNative); + finalizerQueue = gc_.PerformFullGC(); + } + + // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would + // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing + // kotlin code. + + // TODO: These will actually need to be run on a separate thread. + // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`. + AssertThreadState(ThreadState::kRunnable); + finalizerQueue.Finalize(); } void gc::SingleThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { PerformFullGC(); } -void gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept { - RuntimeAssert(running_ == false, "Cannot have been called during another collection"); - running_ = true; +void gc::SingleThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { + size_t counterOverhead = gc_.GetThreshold() == 0 ? safePointsCounter_ : safePointsCounter_ % gc_.GetThreshold(); + if (threadData_.suspensionData().suspendIfRequested()) { + safePointsCounter_ = 0; + } else if (counterOverhead + weight >= gc_.GetThreshold()) { + safePointsCounter_ = 0; + PerformFullGC(); + } + safePointsCounter_ += weight; +} + +mm::ObjectFactory::FinalizerQueue gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept { + bool didSuspend = mm::SuspendThreads(); + if (!didSuspend) { + // Somebody else suspended the threads, and so ran a GC. + // TODO: This breaks if suspension is used by something apart from GC. + return {}; + } KStdVector graySet; for (auto& thread : mm::GlobalData::Instance().threadRegistry().Iter()) { + // TODO: Maybe it's more efficient to do by the suspending thread? thread.Publish(); for (auto* object : mm::ThreadRootSet(thread)) { if (!isNullOrMarker(object)) { @@ -109,10 +137,7 @@ void gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept { gc::Mark(std::move(graySet)); auto finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); - running_ = false; + mm::ResumeThreads(); - // TODO: These will actually need to be run on a separate thread. - // TODO: This probably should check for the existence of runtime itself, but unit tests initialize only memory. - RuntimeAssert(mm::ThreadRegistry::Instance().CurrentThreadData() != nullptr, "Finalizers need a Kotlin runtime"); - finalizerQueue.Finalize(); + return finalizerQueue; } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp index d8eadc6b9d0..566f0edb3ef 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp @@ -8,13 +8,20 @@ #include +#include "ObjectFactory.hpp" #include "Types.h" #include "Utils.hpp" namespace kotlin { + +namespace mm { +class ThreadData; +} + namespace gc { -// Stop-the-world Mark-and-Sweep for a single mutator +// Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads. +// TODO: Rename it away from SingleThreadMarkAndSweep, but keep it STMS. class SingleThreadMarkAndSweep : private Pinned { public: class ObjectData { @@ -36,7 +43,7 @@ public: public: using ObjectData = SingleThreadMarkAndSweep::ObjectData; - explicit ThreadData(SingleThreadMarkAndSweep& gc) noexcept : gc_(gc) {} + explicit ThreadData(SingleThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} ~ThreadData() = default; void SafePointFunctionEpilogue() noexcept; @@ -49,7 +56,10 @@ public: void OnOOM(size_t size) noexcept; private: + void SafePointRegular(size_t weight) noexcept; + SingleThreadMarkAndSweep& gc_; + mm::ThreadData& threadData_; size_t allocatedBytes_ = 0; size_t safePointsCounter_ = 0; }; @@ -67,9 +77,7 @@ public: bool GetAutoTune() noexcept { return autoTune_; } private: - void PerformFullGC() noexcept; - - bool running_ = false; + mm::ObjectFactory::FinalizerQueue PerformFullGC() noexcept; size_t threshold_ = 1000; size_t allocationThresholdBytes_ = 10000; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp index fac94e55dd8..ef4864fe111 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp @@ -5,6 +5,11 @@ #include "SingleThreadMarkAndSweep.hpp" +#include +#include +#include +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -58,6 +63,10 @@ public: mm::AllocateObject(&threadData, typeHolder.typeInfo(), &location_); } + GlobalObjectHolder(mm::ThreadData& threadData, ObjHeader* object) : location_(object) { + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(&threadData, &location_); + } + ObjHeader* header() { return location_; } test_support::Object& operator*() { return test_support::Object::FromObjHeader(location_); } @@ -126,6 +135,7 @@ class StackObjectHolder : private Pinned { public: explicit StackObjectHolder(mm::ThreadData& threadData) { mm::AllocateObject(&threadData, typeHolder.typeInfo(), holder_.slot()); } explicit StackObjectHolder(test_support::Object& object) : holder_(object.header()) {} + explicit StackObjectHolder(ObjHeader* object) : holder_(object) {} ObjHeader* header() { return holder_.obj(); } @@ -629,3 +639,404 @@ TEST_F(SingleThreadMarkAndSweepTest, SameObjectInRootSet) { EXPECT_THAT(GetColor(object.header()), Color::kWhite); }); } + +namespace { + +class Mutator : private Pinned { +public: + Mutator() : thread_(&Mutator::RunLoop, this) {} + + ~Mutator() { + { + std::unique_lock guard(queueMutex_); + shutdownRequested_ = true; + } + queueCV_.notify_one(); + thread_.join(); + RuntimeAssert(queue_.empty(), "The queue must be empty, has size=%zu", queue_.size()); + RuntimeAssert(memory_ == nullptr, "Memory must have been deinitialized"); + RuntimeAssert(stackRoots_.empty(), "Stack roots must be empty, has size=%zu", stackRoots_.size()); + RuntimeAssert(globalRoots_.empty(), "Global roots must be empty, has size=%zu", globalRoots_.size()); + } + + template + [[nodiscard]] std::future Execute(F&& f) { + std::packaged_task task([this, f = std::forward(f)]() { f(*memory_->memoryState()->GetThreadData(), *this); }); + auto future = task.get_future(); + { + std::unique_lock guard(queueMutex_); + queue_.push_back(std::move(task)); + } + queueCV_.notify_one(); + return future; + } + + StackObjectHolder& AddStackRoot() { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData()); + auto& holderRef = *holder; + stackRoots_.push_back(std::move(holder)); + return holderRef; + } + + StackObjectHolder& AddStackRoot(ObjHeader* object) { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddStackRoot can only be called in the mutator thread"); + auto holder = make_unique(object); + auto& holderRef = *holder; + stackRoots_.push_back(std::move(holder)); + return holderRef; + } + + GlobalObjectHolder& AddGlobalRoot() { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData()); + auto& holderRef = *holder; + globalRoots_.push_back(std::move(holder)); + return holderRef; + } + + GlobalObjectHolder& AddGlobalRoot(ObjHeader* object) { + RuntimeAssert(std::this_thread::get_id() == thread_.get_id(), "AddGlobalRoot can only be called in the mutator thread"); + auto holder = make_unique(*memory_->memoryState()->GetThreadData(), object); + auto& holderRef = *holder; + globalRoots_.push_back(std::move(holder)); + return holderRef; + } + + KStdVector Alive() { return ::Alive(*memory_->memoryState()->GetThreadData()); } + +private: + void RunLoop() { + memory_ = make_unique(); + AssertThreadState(memory_->memoryState(), ThreadState::kRunnable); + + while (true) { + std::packaged_task task; + { + std::unique_lock guard(queueMutex_); + queueCV_.wait(guard, [this]() { return !queue_.empty() || shutdownRequested_; }); + if (shutdownRequested_) { + globalRoots_.clear(); + stackRoots_.clear(); + memory_.reset(); + return; + } + task = std::move(queue_.front()); + queue_.pop_front(); + } + task(); + } + } + + KStdUniquePtr memory_; + + // TODO: Consider full runtime init instead, and interact with initialized worker + std::condition_variable queueCV_; + std::mutex queueMutex_; + KStdDeque> queue_; + bool shutdownRequested_ = false; + std::thread thread_; + + KStdVector> globalRoots_; + KStdVector> stackRoots_; +}; + +} // namespace + +TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsCollect) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + local->field1 = reachable.header(); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsAllCollect) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + local->field1 = reachable.header(); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + // TODO: Maybe check that only one GC is performed. + for (int i = 0; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRequested) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(kDefaultThreadCount); + + auto allocateInHeap = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = AllocateObject(threadData); + auto& local = AllocateObject(threadData); + auto& reachable = AllocateObject(threadData); + AllocateObject(threadData); + + local->field1 = reachable.header(); + + globals[i] = global.header(); + locals[i] = local.header(); + reachables[i] = reachable.header(); + }; + + auto expandRootSet = [&globals, &locals](mm::ThreadData& threadData, Mutator& mutator, int i) { + mutator.AddGlobalRoot(globals[i]); + mutator.AddStackRoot(locals[i]); + }; + + mutators[0] + .Execute([expandRootSet, allocateInHeap](mm::ThreadData& threadData, Mutator& mutator) { + allocateInHeap(threadData, mutator, 0); + expandRootSet(threadData, mutator, 0); + }) + .wait(); + + // Allocate everything in heap before scheduling the GC. + for (int i = 1; i < kDefaultThreadCount; ++i) { + mutators[i] + .Execute([allocateInHeap, i](mm::ThreadData& threadData, Mutator& mutator) { allocateInHeap(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { + expandRootSet(threadData, mutator, i); + threadData.gc().SafePointFunctionEpilogue(); + }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SingleThreadMarkAndSweepTest, CrossThreadReference) { + KStdVector mutators(kDefaultThreadCount); + KStdVector globals(kDefaultThreadCount); + KStdVector locals(kDefaultThreadCount); + KStdVector reachables(2 * kDefaultThreadCount); + + auto expandRootSet = [&globals, &locals, &reachables](mm::ThreadData& threadData, Mutator& mutator, int i) { + auto& global = mutator.AddGlobalRoot(); + auto& local = mutator.AddStackRoot(); + auto& reachable1 = AllocateObject(threadData); + auto& reachable2 = AllocateObject(threadData); + globals[i] = global.header(); + locals[i] = local.header(); + reachables[2 * i] = reachable1.header(); + reachables[2 * i + 1] = reachable2.header(); + + // Expected to be run consequtively, so `reachables` for `j < i` are set. + if (i != 0) { + global->field1 = reachables[2 * (i - 1)]; + local->field1 = reachables[2 * (i - 1) + 1]; + } + }; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + // `expandRootSet` is expected to be run consequtively for each thread, so `.wait()` is required below. + mutators[i] + .Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); }) + .wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().PerformFullGC(); }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + KStdVector expectedAlive; + for (auto& global : globals) { + expectedAlive.push_back(global); + } + for (auto& local : locals) { + expectedAlive.push_back(local); + } + // The last two are in fact unreachable. Their absence allows us to check that GC was in fact performed. + reachables.pop_back(); + reachables.pop_back(); + for (auto& reachable : reachables) { + expectedAlive.push_back(reachable); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); + } +} + +TEST_F(SingleThreadMarkAndSweepTest, MultipleMutatorsWeaks) { + KStdVector mutators(kDefaultThreadCount); + ObjHeader* globalRoot = nullptr; + WeakCounter* weak = nullptr; + + mutators[0] + .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { + auto& global = mutator.AddGlobalRoot(); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> WeakCounter& { + ObjHolder holder; + return InstallWeakCounter(threadData, object.header(), holder.slot()); + })(); + global->field1 = objectWeak.header(); + weak = &objectWeak; + globalRoot = global.header(); + }) + .wait(); + + // Make sure all mutators are initialized. + for (int i = 1; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + KStdVector> gcFutures(kDefaultThreadCount); + + gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PerformFullGC(); + EXPECT_THAT((*weak)->referred, nullptr); + }); + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (int i = 1; i < kDefaultThreadCount; ++i) { + gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().SafePointFunctionEpilogue(); + EXPECT_THAT((*weak)->referred, nullptr); + }); + } + + for (auto& future : gcFutures) { + future.wait(); + } + + for (auto& mutator : mutators) { + EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); + } +} + +// TODO: Attaching new threads while GC is in progress. diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index 9288ad7f774..06789e5cb8e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -79,7 +79,10 @@ void mm::ExtraObjectData::ClearWeakReferenceCounter() noexcept { if (!HasWeakReferenceCounter()) return; WeakReferenceCounterClear(weakReferenceCounter_); - mm::SetHeapRef(&weakReferenceCounter_, nullptr); + // Not using `mm::SetHeapRef here`, because this code is called during sweep phase by the GC thread, + // and so cannot affect marking. + // TODO: Asserts on the above? + weakReferenceCounter_ = nullptr; } mm::ExtraObjectData::~ExtraObjectData() { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index d7777567d0a..78b2d11e4f8 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -515,6 +515,7 @@ public: auto* heapObject = new (node.Data()) HeapObjHeader(); auto* object = &heapObject->object; object->typeInfoOrMeta_ = const_cast(typeInfo); + // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. return object; } @@ -528,6 +529,7 @@ public: auto* array = &heapArray->array; array->typeInfoOrMeta_ = const_cast(typeInfo); array->count_ = count; + // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. return array; } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index b6407c466c4..8368d392951 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -33,7 +33,7 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()), - gc_(GlobalData::Instance().gc()), + gc_(GlobalData::Instance().gc(), *this), objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_), suspensionData_(ThreadState::kRunnable) {} diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadState.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadState.cpp index b4c00643b2e..32a487eb4f7 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadState.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadState.cpp @@ -42,4 +42,4 @@ ALWAYS_INLINE void kotlin::AssertThreadState(MemoryState* thread, std::initializ ThreadState kotlin::GetThreadState(MemoryState* thread) noexcept { return thread->GetThreadData()->state(); -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index 50c367bac55..03f5c9455b0 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -33,7 +33,8 @@ kotlin::test_support::TypeInfoHolder theObjCObjectWrapperTypeInfoHolder{ kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; kotlin::test_support::TypeInfoHolder theOpaqueFunctionTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; kotlin::test_support::TypeInfoHolder theShortArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; -kotlin::test_support::TypeInfoHolder theStringTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; +kotlin::test_support::TypeInfoHolder theStringTypeInfoHolder{ + kotlin::test_support::TypeInfoHolder::ArrayBuilder().addFlag(TF_IMMUTABLE)}; kotlin::test_support::TypeInfoHolder theThrowableTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; kotlin::test_support::TypeInfoHolder theUnitTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; kotlin::test_support::TypeInfoHolder theWorkerBoundReferenceTypeInfoHolder{