KT-56789: Fix memory leak in CoreJrtFileSystem

CoreJrtFileSystem uses JrtFileSystemProvider provider to read contents
of jrt-fs from JDK
Implementation of FileSystems.newFileSystem causes metaspace memory leak
that wasn't fixed until JDK 17, see
https://bugs.openjdk.java.net/browse/JDK-8260621

When FileSystems.newFileSystem used to create jrt-fs on JDK9 with
provided java.home value it creates new ClassLoader under-the-hood,
which subsequently leaks due to aforementioned bug

Remove conditional usage of `java.home` + FileSystems.newFileSystem and
switch to use jrt-fs classloader cache regardless of
compiler runtime JDK to reduce classloader leaks

^KT-56789

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/modules/CoreJrtFileSystem.kt

@@ -34,20 +34,17 @@ class CoreJrtFileSystem : DeprecatedVirtualFileSystem() {
             val jdkHome = File(jdkHomePath)
             val rootUri = URI.create(StandardFileSystems.JRT_PROTOCOL + ":/")
             val jrtFsJar = loadJrtFsJar(jdkHome) ?: return@createMap null
-            val fileSystem =
+
-                if (isAtLeastJava9()) {
+            /*
-                    FileSystems.newFileSystem(rootUri, mapOf("java.home" to jdkHome.absolutePath))
+              This ClassLoader actually lives as long as current thread due to ThreadLocal leak in jrt-fs,
-                } else {
+              See https://bugs.openjdk.java.net/browse/JDK-8260621
-                    /*
+              So that cache allows us to avoid creating too many classloaders for same JDK and reduce severity of that leak
-                    This ClassLoader actually lives as long as current thread due to ThreadLocal leak in jrtfs,
+            */
-                    See https://bugs.openjdk.java.net/browse/JDK-8260621
+            val classLoader = globalJrtFsClassLoaderCache.computeIfAbsent(jrtFsJar) {
-                    So that cache allows us to avoid creating too many classloaders for same JDK and reduce severity of that leak
+                URLClassLoader(arrayOf(jrtFsJar.toURI().toURL()), null)
-                    */
+            }
-                    val classLoader = jrtFsClassLoaderCache.computeIfAbsent(jrtFsJar) {
+
-                        URLClassLoader(arrayOf(jrtFsJar.toURI().toURL()), null)
+            val fileSystem = FileSystems.newFileSystem(rootUri, emptyMap<String, Nothing>(), classLoader)
-                    }
-                    FileSystems.newFileSystem(rootUri, emptyMap<String, Nothing>(), classLoader)
-                }
             CoreJrtVirtualFile(this, jdkHomePath, fileSystem.getPath(""), parent = null)
         }
 
@@ -87,6 +84,6 @@ class CoreJrtFileSystem : DeprecatedVirtualFileSystem() {
             return Pair(localPath, pathInJar)
         }
 
-        private val jrtFsClassLoaderCache = ContainerUtil.createConcurrentWeakValueMap<File, URLClassLoader>()
+        private val globalJrtFsClassLoaderCache = ContainerUtil.createConcurrentWeakValueMap<File, URLClassLoader>()
     }
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt

@@ -136,6 +136,24 @@ class KotlinDaemonIT : KGPDaemonsBaseTest() {
         }
     }
 
+    @DisplayName("KT-56789: Kotlin daemon does not triggers OOM in Metaspace on multiple invocations")
+    @JdkVersions(versions = [JavaVersion.VERSION_11])
+    @GradleWithJdkTest
+    @GradleTestVersions(minVersion = TestVersions.Gradle.MAX_SUPPORTED)
+    fun testMultipleCompilations(gradleVersion: GradleVersion, jdk: JdkVersions.ProvidedJdk) {
+        project(
+            "daemonJvmResourceLimits",
+            gradleVersion,
+            buildJdk = jdk.location
+        ) {
+            for (iteration in 0..300) {
+                build("clean", "assemble") {
+                    assertKotlinDaemonReusesOnlyOneSession()
+                }
+            }
+        }
+    }
+
 
     private fun BuildResult.assertGradleClasspathNotLeaked() {
         assertOutputContains("Kotlin compiler classpath:")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/daemonJvmResourceLimits/build.gradle.kts

@@ -0,0 +1,10 @@
+plugins {
+    kotlin("jvm")
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+dependencies {}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/daemonJvmResourceLimits/gradle.properties

@@ -0,0 +1 @@
+kotlin.daemon.jvmargs=-XX:MaxMetaspaceSize=128m -Xmx256m

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/daemonJvmResourceLimits/src/main/kotlin/helloWorld.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+fun main() {
+    System.out.println("Hello, world!")
+    println("Hello, world!")
+}