Preserve FastJarFS in app env and implement intermediate cleanup

FastJarFS was behaving differently than the regular CoreJarFS it
tries to replace, namely it was cleaned after each compilation
(had the lifetime tied to core env).
This commit preserve it in the app env the same way as CoreJarFS,
so it could be reused in parallel builds and preserved if automatic
cleanup of the app env is turned off,
But since FastJarFS caches also mmf handles, the additional handles
cleaning is introduced that triggers after all possibly parallel
compilations are finished.

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreApplicationEnvironment.kt

@@ -11,27 +11,59 @@ import com.intellij.core.JavaCoreApplicationEnvironment
 import com.intellij.lang.MetaLanguage
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.vfs.VirtualFileSystem
 import com.intellij.psi.FileContextProvider
 import com.intellij.psi.augment.PsiAugmentProvider
 import com.intellij.psi.impl.smartPointers.SmartPointerAnchorProvider
 import com.intellij.psi.meta.MetaDataContributor
 import org.jetbrains.kotlin.cli.jvm.compiler.IdeaExtensionPoints.registerVersionSpecificAppExtensionPoints
+import org.jetbrains.kotlin.cli.jvm.compiler.jarfs.FastJarFileSystem
 import org.jetbrains.kotlin.cli.jvm.modules.CoreJrtFileSystem
 
-class KotlinCoreApplicationEnvironment private constructor(parentDisposable: Disposable, unitTestMode: Boolean) :
+class KotlinCoreApplicationEnvironment private constructor(
+    parentDisposable: Disposable, unitTestMode: Boolean
+) :
     JavaCoreApplicationEnvironment(parentDisposable, unitTestMode) {
-    override fun createJrtFileSystem(): VirtualFileSystem? {
+
+    override fun createJrtFileSystem(): VirtualFileSystem {
         return CoreJrtFileSystem()
     }
 
+    private var fastJarFileSystemField: FastJarFileSystem? = null
+    private var fastJarFileSystemFieldInitialized = false
+
+    val fastJarFileSystem: FastJarFileSystem?
+        get() {
+            synchronized(KotlinCoreEnvironment.APPLICATION_LOCK) {
+                if (!fastJarFileSystemFieldInitialized) {
+
+                    // may return null e.g. on the old JDKs, therefore fastJarFileSystemFieldInitialized flag is needed
+                    fastJarFileSystemField = FastJarFileSystem.createIfUnmappingPossible()?.also {
+                        Disposer.register(parentDisposable) {
+                            it.clearHandlersCache()
+                        }
+                    }
+                    fastJarFileSystemFieldInitialized = true
+                }
+                return fastJarFileSystemField
+            }
+        }
+
+    fun idleCleanup() {
+        fastJarFileSystemField?.cleanOpenFilesCache()
+    }
+
     companion object {
-        fun create(parentDisposable: Disposable, unitTestMode: Boolean): KotlinCoreApplicationEnvironment {
+        fun create(
+            parentDisposable: Disposable, unitTestMode: Boolean
+        ): KotlinCoreApplicationEnvironment {
             val environment = KotlinCoreApplicationEnvironment(parentDisposable, unitTestMode)
             registerExtensionPoints()
             return environment
         }
 
+        @Suppress("UnstableApiUsage")
         private fun registerExtensionPoints() {
             registerApplicationExtensionPoint(DynamicBundle.LanguageBundleEP.EP_NAME, DynamicBundle.LanguageBundleEP::class.java)
             registerApplicationExtensionPoint(FileContextProvider.EP_NAME, FileContextProvider::class.java)

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -60,7 +60,6 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.STRONG_WARNING
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.common.toBooleanLenient
-import org.jetbrains.kotlin.cli.jvm.compiler.jarfs.FastJarFileSystem
 import org.jetbrains.kotlin.cli.jvm.config.*
 import org.jetbrains.kotlin.cli.jvm.index.*
 import org.jetbrains.kotlin.cli.jvm.javac.JavacWrapperRegistrar
@@ -134,22 +133,14 @@ class KotlinCoreEnvironment private constructor(
                     applicationEnvironment.jarFileSystem
                 }
                 configuration.getBoolean(JVMConfigurationKeys.USE_FAST_JAR_FILE_SYSTEM) || configuration.getBoolean(CommonConfigurationKeys.USE_FIR) -> {
-                    val fastJarFs = FastJarFileSystem.createIfUnmappingPossible()
+                    val fastJarFs = applicationEnvironment.fastJarFileSystem
-
                     if (fastJarFs == null) {
                         messageCollector?.report(
-                            STRONG_WARNING,
+                            CompilerMessageSeverity.STRONG_WARNING,
                             "Your JDK doesn't seem to support mapped buffer unmapping, so the slower (old) version of JAR FS will be used"
                         )
                         applicationEnvironment.jarFileSystem
-                    } else {
+                    } else fastJarFs
-
-                        Disposer.register(disposable) {
-                            fastJarFs.clearHandlersCache()
-                        }
-
-                        fastJarFs
-                    }
                 }
 
                 else -> applicationEnvironment.jarFileSystem
@@ -464,7 +455,10 @@ class KotlinCoreEnvironment private constructor(
         ): KotlinCoreEnvironment {
             val configuration = initialConfiguration.copy()
             // Tests are supposed to create a single project and dispose it right after use
-            val appEnv = createApplicationEnvironment(parentDisposable, configuration, unitTestMode = true)
+            val appEnv =
+                createApplicationEnvironment(
+                    parentDisposable, configuration, unitTestMode = true
+                )
             val projectEnv = ProjectEnvironment(parentDisposable, appEnv, configuration)
             return KotlinCoreEnvironment(projectEnv, configuration, extensionConfigs)
         }
@@ -479,7 +473,11 @@ class KotlinCoreEnvironment private constructor(
 
         @TestOnly
         fun createProjectEnvironmentForTests(parentDisposable: Disposable, configuration: CompilerConfiguration): ProjectEnvironment {
-            val appEnv = createApplicationEnvironment(parentDisposable, configuration, unitTestMode = true)
+            val appEnv = createApplicationEnvironment(
+                parentDisposable,
+                configuration,
+                unitTestMode = true
+            )
             return ProjectEnvironment(parentDisposable, appEnv, configuration)
         }
 
@@ -500,7 +498,12 @@ class KotlinCoreEnvironment private constructor(
             synchronized(APPLICATION_LOCK) {
                 if (ourApplicationEnvironment == null) {
                     val disposable = Disposer.newDisposable("Disposable for the KotlinCoreApplicationEnvironment")
-                    ourApplicationEnvironment = createApplicationEnvironment(disposable, configuration, unitTestMode)
+                    ourApplicationEnvironment =
+                        createApplicationEnvironment(
+                            disposable,
+                            configuration,
+                            unitTestMode
+                        )
                     ourProjectCount = 0
                     Disposer.register(disposable, Disposable {
                         synchronized(APPLICATION_LOCK) {
@@ -509,26 +512,28 @@ class KotlinCoreEnvironment private constructor(
                     })
                 }
                 try {
-                    // Do not use this property unless you sure need it, causes Application to MEMORY LEAK
+                    // Disposer uses identity of passed object to deduplicate registered disposables
-                    // Only valid use-case is when Application should be cached to avoid
+                    // We should everytime pass new instance to avoid un-registering from previous one
-                    // initialization costs
+                    @Suppress("ObjectLiteralToLambda")
-                    if (CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value.toBooleanLenient() != true) {
+                    Disposer.register(parentDisposable, object : Disposable {
-                        // Disposer uses identity of passed object to deduplicate registered disposables
+                        override fun dispose() {
-                        // We should everytime pass new instance to avoid un-registering from previous one
+                            synchronized(APPLICATION_LOCK) {
-                        @Suppress("ObjectLiteralToLambda")
+                                // Build-systems may run many instances of the compiler in parallel
-                        Disposer.register(parentDisposable, object : Disposable {
+                                // All projects share the same ApplicationEnvironment, and when the last project is disposed,
-                            override fun dispose() {
+                                // the ApplicationEnvironment is disposed as well
-                                synchronized(APPLICATION_LOCK) {
+                                if (--ourProjectCount <= 0) {
-                                    // Build-systems may run many instances of the compiler in parallel
+                                    // Do not use this property unless you sure need it, causes Application to MEMORY LEAK
-                                    // All projects share the same ApplicationEnvironment, and when the last project is disposed,
+                                    // Only valid use-case is when Application should be cached to avoid
-                                    // the ApplicationEnvironment is disposed as well
+                                    // initialization costs
-                                    if (--ourProjectCount <= 0) {
+                                    if (CompilerSystemProperties.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY.value.toBooleanLenient() != true) {
                                         disposeApplicationEnvironment()
+                                    } else {
+                                        ourApplicationEnvironment?.idleCleanup()
                                     }
                                 }
                             }
-                        })
+                        }
-                    }
+                    })
                 } finally {
                     ourProjectCount++
                 }
@@ -578,7 +583,9 @@ class KotlinCoreEnvironment private constructor(
         private fun createApplicationEnvironment(
             parentDisposable: Disposable, configuration: CompilerConfiguration, unitTestMode: Boolean
         ): KotlinCoreApplicationEnvironment {
-            val applicationEnvironment = KotlinCoreApplicationEnvironment.create(parentDisposable, unitTestMode)
+            val applicationEnvironment = KotlinCoreApplicationEnvironment.create(
+                parentDisposable, unitTestMode
+            )
 
             registerApplicationExtensionPointsAndExtensionsFrom(configuration, "extensions/compiler.xml")
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/jarfs/FastJarFileSystem.kt

@@ -58,6 +58,10 @@ class FastJarFileSystem private constructor(internal val unmapBuffer: MappedByte
 
     fun clearHandlersCache() {
         myHandlers.clear()
+        cleanOpenFilesCache()
+    }
+
+    fun cleanOpenFilesCache() {
         cachedOpenFileHandles.clear()
     }