[Gradle] Introduce persistent storage dir in 'user.home/.kotlin'

This storage directory should be used for:
- files that should live between builds
- files that should stay after calling 'clean' task
- some persistent caches that are usually better to keep around

In this commit, we are introducing the first cache type - for session
flag files. Such files should be unique per-project. To distinguish
between different projects, md5 hash of the project root dir absolute
path is used.

^KT-58223 In Progress

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt

@@ -17,6 +17,7 @@ import kotlin.io.path.*
 import kotlin.test.assertTrue
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.testbase.TestVersions.ThirdPartyDependencies.GRADLE_ENTERPRISE_PLUGIN_VERSION
+import org.junit.jupiter.api.io.TempDir
 
 @DisplayName("Build reports")
 @JvmGradlePluginTests
@@ -263,17 +264,27 @@ class BuildReportsIT : KGPBaseTest() {
         }
     }
 
-    private val kotlinErrorPath = ".gradle/kotlin/errors"
-
     @DisplayName("Error file is created")
     @GradleTestVersions(
         additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
     )
     @GradleTest
-    fun testErrorsFileSmokeTest(gradleVersion: GradleVersion) {
+    fun testErrorsFileSmokeTest(
-        project("simpleProject", gradleVersion) {
+        gradleVersion: GradleVersion,
+        @TempDir tempDir: Path,
+    ) {
+        project(
+            projectName = "simpleProject",
+            gradleVersion = gradleVersion,
+            buildOptions = defaultBuildOptions.copy(kotlinUserHome = tempDir)
+        ) {
 
             val lookupsTab = projectPath.resolve("build/kotlin/compileKotlin/cacheable/caches-jvm/lookups/lookups.tab")
+            fun kotlinErrorPath() = tempDir
+                .resolve("projects-1")
+                .findInPath("errors")
+                ?: throw IllegalStateException("Could not find global Kotlin errors directory!")
+
             buildGradle.appendText(
                 """
                     tasks.named("compileKotlin") {
@@ -283,16 +294,18 @@ class BuildReportsIT : KGPBaseTest() {
                     }
                 """.trimIndent()
             )
-            build("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
+
-                assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
+            build("compileKotlin") {
+                assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
                 assertOutputDoesNotContain("errors were stored into file")
             }
             val kotlinFile = kotlinSourcesDir().resolve("helloWorld.kt")
             kotlinFile.modify { it.replace("ArrayList", "skjfghsjk") }
             buildAndFail("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
                 assertOutputContains("errors were stored into file")
-                val file = projectPath.getSingleFileInDir(kotlinErrorPath)
+                val files = kotlinErrorPath().listDirectoryEntries()
-                file.bufferedReader().use { reader ->
+                assertTrue { files.first().exists() }
+                files.first().bufferedReader().use { reader ->
                     val kotlinVersion = reader.readLine()
                     assertTrue("kotlin version should be in the error file") {
                         kotlinVersion != null && kotlinVersion.trim().equals("kotlin version: ${buildOptions.kotlinVersion}")
@@ -312,17 +325,29 @@ class BuildReportsIT : KGPBaseTest() {
         additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
     )
     @GradleTest
-    fun testErrorsFileWithCompilationError(gradleVersion: GradleVersion) {
+    fun testErrorsFileWithCompilationError(
-        project("simpleProject", gradleVersion) {
+        gradleVersion: GradleVersion,
+        @TempDir tempDir: Path,
+    ) {
+        project(
+            projectName = "simpleProject",
+            gradleVersion = gradleVersion,
+            buildOptions = defaultBuildOptions.copy(kotlinUserHome = tempDir)
+        ) {
+            fun kotlinErrorPath() = tempDir
+                .resolve("projects-1")
+                .findInPath("errors")
+                ?: throw IllegalStateException("Could not find global Kotlin errors directory!")
+
             build("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
-                assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
                 assertOutputDoesNotContain("errors were stored into file")
+                assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
             }
             val kotlinFile = kotlinSourcesDir().resolve("helloWorld.kt")
             kotlinFile.modify { it.replace("ArrayList", "skjfghsjk") }
             buildAndFail("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
-                assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
                 assertOutputDoesNotContain("errors were stored into file")
+                assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
             }
         }
     }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -25,7 +25,6 @@ import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions
 import org.jetbrains.kotlin.daemon.common.filterExtractProps
 import org.jetbrains.kotlin.gradle.dsl.KotlinJsProjectExtension
 import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
-import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
 import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
@@ -47,6 +46,9 @@ import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
 import java.io.File
 import java.lang.ref.WeakReference
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import kotlin.concurrent.read
+import kotlin.concurrent.write
 
 
 const val CREATED_CLIENT_FILE_PREFIX = "Created client-is-alive flag file: "
@@ -105,7 +107,6 @@ internal open class GradleCompilerRunner(
     internal val loggerProvider = taskProvider.logger.get()
     internal val buildDirProvider = taskProvider.buildDir.get().asFile
     internal val projectDirProvider = taskProvider.projectDir.get()
-    internal val projectCacheDirProvider = taskProvider.projectCacheDir.get()
     internal val sessionDirProvider = taskProvider.sessionsDir.get()
     internal val projectNameProvider = taskProvider.projectName.get()
     internal val incrementalModuleInfoProvider = taskProvider.buildModulesInfo
@@ -216,7 +217,6 @@ internal open class GradleCompilerRunner(
                 projectDirProvider,
                 buildDirProvider,
                 projectNameProvider,
-                projectCacheDirProvider,
                 sessionDirProvider
             ),
             compilerFullClasspath = environment.compilerFullClasspath(jdkToolsJar),
@@ -456,25 +456,44 @@ internal open class GradleCompilerRunner(
         }
 
         // session is created per build
-        @Volatile
         private var sessionFlagFile: File? = null
 
+        private val sessionFileLock = ReentrantReadWriteLock(true)
+
         // session files are deleted at org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices.buildFinished
-        @Synchronized
+
-        internal fun getOrCreateSessionFlagFile(log: Logger, sessionsDir: File, projectCacheDirProvider: File): File {
+        internal fun getOrCreateSessionFlagFile(
-            if (sessionFlagFile == null || !sessionFlagFile!!.exists()) {
+            log: Logger,
-                val sessionFilesDir = sessionsDir.apply { mkdirs() }
+            sessionsDir: File,
-                sessionFlagFile = newTmpFile(prefix = "kotlin-compiler-", suffix = ".salive", directory = sessionFilesDir)
+        ): File {
-                log.kotlinDebug { CREATED_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeOrAbsolute(projectCacheDirProvider) }
+            sessionFileLock.read {
-            } else {
+                val sessionFlagRead = sessionFlagFile
-                log.kotlinDebug { EXISTING_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeOrAbsolute(projectCacheDirProvider) }
+                if (sessionFlagRead != null && sessionFlagRead.exists()) {
+                    return sessionFlagRead.sessionFileFlagExists(log)
+                }
             }
 
-            return sessionFlagFile!!
+            sessionFileLock.write {
+                val sessionFlagWrite = sessionFlagFile
+                if (sessionFlagWrite != null && sessionFlagWrite.exists()) {
+                    return sessionFlagWrite.sessionFileFlagExists(log)
+                }
+
+                return newTmpFile(
+                    prefix = "kotlin-compiler-",
+                    suffix = ".salive",
+                    directory = sessionsDir.apply { mkdirs() })
+                    .also {
+                        sessionFlagFile = it
+                        log.kotlinDebug { CREATED_SESSION_FILE_PREFIX + it.absolutePath }
+                    }
+            }
         }
 
-        internal fun sessionsDir(projectCacheDir: File): File =
+        private fun File.sessionFileFlagExists(log: Logger): File {
-            File(File(projectCacheDir, "kotlin"), "sessions")
+            log.kotlinDebug { EXISTING_SESSION_FILE_PREFIX + absolutePath }
+            return this
+        }
     }
 }
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt

@@ -42,10 +42,16 @@ internal class ProjectFilesForCompilation(
     val sessionFlagFile: File,
     val buildDir: File
 ) : Serializable {
-    constructor(logger: Logger, projectDir:File, buildDir: File, projectName: String, projectCacheDirProvider: File, sessionDir: File) : this(
+    constructor(
+        logger: Logger,
+        projectDir: File,
+        buildDir: File,
+        projectName: String,
+        sessionDir: File
+    ) : this(
         projectRootFile = projectDir,
         clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, projectName),
-        sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir, projectCacheDirProvider),
+        sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir),
         buildDir = buildDir
     )
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt

@@ -8,17 +8,18 @@ package org.jetbrains.kotlin.gradle.plugin
 import org.gradle.api.Project
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.logging.Logging
+import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
 import org.gradle.api.services.BuildService
 import org.gradle.api.services.BuildServiceParameters
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
-import org.jetbrains.kotlin.gradle.utils.projectCacheDir
+import org.jetbrains.kotlin.gradle.utils.kotlinSessionsDir
 import java.io.File
 
 internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBuildServices.Parameters>, AutoCloseable {
 
     interface Parameters : BuildServiceParameters {
-        var projectCacheDir: File
+        val sessionsDir: Property<File>
     }
 
     private val log = Logging.getLogger(this.javaClass)
@@ -61,7 +62,7 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
     }
 
     override fun close() {
-        buildHandler.buildFinished(parameters.projectCacheDir)
+        buildHandler.buildFinished(parameters.sessionsDir.get())
         log.kotlinDebug(DISPOSE_MESSAGE)
     }
 
@@ -75,7 +76,7 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
                 "kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}",
                 KotlinGradleBuildServices::class.java
             ) { service ->
-                service.parameters.projectCacheDir = gradle.projectCacheDir
+                service.parameters.sessionsDir.set(gradle.rootProject.kotlinSessionsDir)
             }
 
     }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleFinishBuildHandler.kt

@@ -32,24 +32,23 @@ internal class KotlinGradleFinishBuildHandler {
         startMemory = getUsedMemoryKb()
     }
 
-    fun buildFinished(projectCacheDir: File) {
+    fun buildFinished(sessionsDir: File) {
         TaskLoggers.clear()
         TaskExecutionResults.clear()
 
         GradleCompilerRunner.clearBuildModulesInfo()
 
-        val sessionsDir = GradleCompilerRunner.sessionsDir(projectCacheDir)
         if (sessionsDir.exists()) {
             val sessionFiles = sessionsDir.listFiles()
 
             // it is expected that only one session file per build exists
             // afaik is is not possible to run multiple gradle builds in one project since gradle locks some dirs
             if (sessionFiles.size > 1) {
-                log.warn("w: Detected multiple Kotlin daemon sessions at ${sessionsDir.relativeOrAbsolute(projectCacheDir)}")
+                log.warn("w: Detected multiple Kotlin daemon sessions at ${sessionsDir.relativeOrAbsolute(sessionsDir)}")
             }
             for (file in sessionFiles) {
                 file.delete()
-                log.kotlinDebug { DELETED_SESSION_FILE_PREFIX + file.relativeOrAbsolute(projectCacheDir) }
+                log.kotlinDebug { DELETED_SESSION_FILE_PREFIX + file.absolutePath }
             }
         }
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt

@@ -543,10 +543,9 @@ internal class PropertiesProvider private constructor(private val project: Proje
      * This data is not cleaned by gradle clean task. This dir can be used to store data
      * for IDE import i.e. `./gradlew clean` will not break IDE import state.
      *
-     * If the property is not set, the default value `$ROOT_DIR/.kotlin` is used.
+     * If the property is not set, the plugin will use `<user_home>/.kotlin` as default.
      */
-    val kotlinPersistentGradleDataDir: String
+    val kotlinPersistentGradleDataDir: String? = get(PropertyNames.KOTLIN_PERSISTENT_GRADLE_DATA_DIR)
-        get() = get(PropertyNames.KOTLIN_PERSISTENT_GRADLE_DATA_DIR) ?: ".kotlin"
 
     /**
      * Retrieves a comma-separated list of browsers to use when running karma tests for [target]

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt

@@ -18,9 +18,12 @@ private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransforme
 internal fun ProjectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
     buildDirectory.get().asFile.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)
 
+internal val Project.kotlinGradleDataDirCompat
+    get() = kotlinPropertiesProvider.kotlinPersistentGradleDataDir ?: ".kotlin"
+
 internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File
     get() = compositeBuildRootProject.rootDir
-        .resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
+        .resolve(project.kotlinGradleDataDirCompat)
         .resolve(kotlinTransformedMetadataLibraries)
 
 internal fun ProjectLayout.kotlinTransformedCInteropMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
@@ -28,10 +31,10 @@ internal fun ProjectLayout.kotlinTransformedCInteropMetadataLibraryDirectoryForB
 
 internal val Project.kotlinCInteropLibraryDirectoryForIde: File
     get() = compositeBuildRootProject.rootDir
-        .resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
+        .resolve(project.kotlinGradleDataDirCompat)
         .resolve(kotlinCInteropLibraries)
 
 internal val Project.kotlinTransformedCInteropMetadataLibraryDirectoryForIde: File
     get() = compositeBuildRootProject.rootDir
-        .resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
+        .resolve(project.kotlinGradleDataDirCompat)
         .resolve(kotlinTransformedCInteropMetadataLibraries)

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CopyCInteropCommonizerTaskOutputForIdeTask.kt

@@ -16,7 +16,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.compilerRunner.addBuildMetricsForTaskAction
 import org.jetbrains.kotlin.gradle.report.GradleBuildMetricsReporter
 import org.jetbrains.kotlin.gradle.utils.property
-import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
+import org.jetbrains.kotlin.gradle.plugin.mpp.kotlinGradleDataDirCompat
 import java.io.File
 import javax.inject.Inject
 
@@ -34,7 +34,7 @@ internal abstract class CopyCommonizeCInteropForIdeTask @Inject constructor(
 
     @get:OutputDirectory
     override val outputDirectory: File = project.rootDir
-        .resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
+        .resolve(project.kotlinGradleDataDirCompat)
         .resolve("commonizer")
         .resolve(project.path.removePrefix(":").replace(":", "/"))
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleCompileTaskProvider.kt

@@ -14,11 +14,10 @@ import org.gradle.api.logging.Logger
 import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Internal
-import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
 import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner.Companion.normalizeForFlagFile
 import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoProvider
-import org.jetbrains.kotlin.gradle.utils.projectCacheDir
 import org.jetbrains.kotlin.gradle.utils.property
+import org.jetbrains.kotlin.gradle.utils.kotlinSessionsDir
 import java.io.File
 import javax.inject.Inject
 
@@ -44,13 +43,9 @@ abstract class GradleCompileTaskProvider @Inject constructor(
     val projectDir: Provider<File> = objectFactory
         .property(project.rootProject.projectDir)
 
-    @get:Internal
-    val projectCacheDir: Provider<File> = objectFactory
-        .property(gradle.projectCacheDir)
-
     @get:Internal
     val sessionsDir: Provider<File> = objectFactory
-        .property(GradleCompilerRunner.sessionsDir(gradle.projectCacheDir))
+        .property(project.kotlinSessionsDir)
 
     @get:Internal
     val projectName: Provider<String> = objectFactory

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/gradleUtils.kt

@@ -18,9 +18,6 @@ import org.gradle.api.services.BuildServiceSpec
 import org.gradle.util.GradleVersion
 import kotlin.reflect.KClass
 
-val Gradle.projectCacheDir
-    get() = startParameter.projectCacheDir ?: this.rootProject.projectDir.resolve(".gradle")
-
 internal val Project.compositeBuildRootGradle: Gradle get() = generateSequence(project.gradle) { it.parent }.last()
 internal val Project.compositeBuildRootProject: Project get() = compositeBuildRootGradle.rootProject
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/persistentCaches.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.utils
+
+import org.gradle.api.Project
+import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
+import java.io.File
+import java.security.MessageDigest
+
+private const val PROJECTS_CACHE_NAME = "projects"
+private const val PROJECTS_CACHE_VERSION = 1
+private const val PROJECTS_CACHE_NAME_FULL = "$PROJECTS_CACHE_NAME-$PROJECTS_CACHE_VERSION"
+
+private const val SESSIONS_DIR_NAME = "sessions"
+
+internal val Project.basePersistentDir
+    get() = kotlinPropertiesProvider.kotlinPersistentGradleDataDir?.let { File(it) }
+        ?: File(System.getProperty("user.home")).resolve(".kotlin")
+
+internal val Project.kotlinSessionsDir
+    get() = basePersistentDir.resolve(PROJECTS_CACHE_NAME_FULL).resolve(rootDir.absolutePathMd5Hash()).resolve(SESSIONS_DIR_NAME)
+
+private val md5Digest by lazy { MessageDigest.getInstance("MD5") }
+
+private fun File.absolutePathMd5Hash(): String = md5Digest.digest(absolutePath.toByteArray()).toHexString()