azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Gradle] Introduce persistent storage dir in 'user.home/.kotlin'

Browse Source 
This storage directory should be used for:
- files that should live between builds
- files that should stay after calling 'clean' task
- some persistent caches that are usually better to keep around

In this commit, we are introducing the first cache type - for session
flag files. Such files should be unique per-project. To distinguish
between different projects, md5 hash of the project root dir absolute
path is used.

^KT-58223 In Progress
This commit is contained in:
Yahor Berdnikau
2023-08-14 22:24:26 +02:00
committed by Space Team
parent 73c76c5b2d
commit 0d57436bce
11 changed files with 127 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt
+37 -12
View File
@@ -17,6 +17,7 @@ import kotlin.io.path.*
import kotlin.test.assertTrue
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.testbase.TestVersions.ThirdPartyDependencies.GRADLE_ENTERPRISE_PLUGIN_VERSION
import org.junit.jupiter.api.io.TempDir
@DisplayName("Build reports")
@JvmGradlePluginTests
@@ -263,17 +264,27 @@ class BuildReportsIT : KGPBaseTest() {
}
}
private val kotlinErrorPath = ".gradle/kotlin/errors"
@DisplayName("Error file is created")
@GradleTestVersions(
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
)
@GradleTest
fun testErrorsFileSmokeTest(gradleVersion: GradleVersion) {
project("simpleProject", gradleVersion) {
fun testErrorsFileSmokeTest(
gradleVersion: GradleVersion,
@TempDir tempDir: Path,
) {
project(
projectName = "simpleProject",
gradleVersion = gradleVersion,
buildOptions = defaultBuildOptions.copy(kotlinUserHome = tempDir)
) {
val lookupsTab = projectPath.resolve("build/kotlin/compileKotlin/cacheable/caches-jvm/lookups/lookups.tab")
fun kotlinErrorPath() = tempDir
.resolve("projects-1")
.findInPath("errors")
?: throw IllegalStateException("Could not find global Kotlin errors directory!")
buildGradle.appendText(
"""
tasks.named("compileKotlin") {
@@ -283,16 +294,18 @@ class BuildReportsIT : KGPBaseTest() {
}
""".trimIndent()
)
build("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
build("compileKotlin") {
assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
assertOutputDoesNotContain("errors were stored into file")
}
val kotlinFile = kotlinSourcesDir().resolve("helloWorld.kt")
kotlinFile.modify { it.replace("ArrayList", "skjfghsjk") }
buildAndFail("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
assertOutputContains("errors were stored into file")
val file = projectPath.getSingleFileInDir(kotlinErrorPath)
file.bufferedReader().use { reader ->
val files = kotlinErrorPath().listDirectoryEntries()
assertTrue { files.first().exists() }
files.first().bufferedReader().use { reader ->
val kotlinVersion = reader.readLine()
assertTrue("kotlin version should be in the error file") {
kotlinVersion != null && kotlinVersion.trim().equals("kotlin version: ${buildOptions.kotlinVersion}")
@@ -312,17 +325,29 @@ class BuildReportsIT : KGPBaseTest() {
additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0],
)
@GradleTest
fun testErrorsFileWithCompilationError(gradleVersion: GradleVersion) {
project("simpleProject", gradleVersion) {
fun testErrorsFileWithCompilationError(
gradleVersion: GradleVersion,
@TempDir tempDir: Path,
) {
project(
projectName = "simpleProject",
gradleVersion = gradleVersion,
buildOptions = defaultBuildOptions.copy(kotlinUserHome = tempDir)
) {
fun kotlinErrorPath() = tempDir
.resolve("projects-1")
.findInPath("errors")
?: throw IllegalStateException("Could not find global Kotlin errors directory!")
build("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
assertOutputDoesNotContain("errors were stored into file")
assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
}
val kotlinFile = kotlinSourcesDir().resolve("helloWorld.kt")
kotlinFile.modify { it.replace("ArrayList", "skjfghsjk") }
buildAndFail("compileKotlin", buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG)) {
assertTrue { projectPath.resolve(kotlinErrorPath).listDirectoryEntries().isEmpty() }
assertOutputDoesNotContain("errors were stored into file")
assertTrue { kotlinErrorPath().listDirectoryEntries().isEmpty() }
}
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt
+34 -15
View File
@@ -25,7 +25,6 @@ import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions
import org.jetbrains.kotlin.daemon.common.filterExtractProps
import org.jetbrains.kotlin.gradle.dsl.KotlinJsProjectExtension
import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
@@ -47,6 +46,9 @@ import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
import org.jetbrains.kotlin.statistics.metrics.StringMetrics
import java.io.File
import java.lang.ref.WeakReference
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.read
import kotlin.concurrent.write
const val CREATED_CLIENT_FILE_PREFIX = "Created client-is-alive flag file: "
@@ -105,7 +107,6 @@ internal open class GradleCompilerRunner(
internal val loggerProvider = taskProvider.logger.get()
internal val buildDirProvider = taskProvider.buildDir.get().asFile
internal val projectDirProvider = taskProvider.projectDir.get()
internal val projectCacheDirProvider = taskProvider.projectCacheDir.get()
internal val sessionDirProvider = taskProvider.sessionsDir.get()
internal val projectNameProvider = taskProvider.projectName.get()
internal val incrementalModuleInfoProvider = taskProvider.buildModulesInfo
@@ -216,7 +217,6 @@ internal open class GradleCompilerRunner(
projectDirProvider,
buildDirProvider,
projectNameProvider,
projectCacheDirProvider,
sessionDirProvider
),
compilerFullClasspath = environment.compilerFullClasspath(jdkToolsJar),
@@ -456,25 +456,44 @@ internal open class GradleCompilerRunner(
}
// session is created per build
@Volatile
private var sessionFlagFile: File? = null
private val sessionFileLock = ReentrantReadWriteLock(true)
// session files are deleted at org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices.buildFinished
@Synchronized
internal fun getOrCreateSessionFlagFile(log: Logger, sessionsDir: File, projectCacheDirProvider: File): File {
if (sessionFlagFile == null || !sessionFlagFile!!.exists()) {
val sessionFilesDir = sessionsDir.apply { mkdirs() }
sessionFlagFile = newTmpFile(prefix = "kotlin-compiler-", suffix = ".salive", directory = sessionFilesDir)
log.kotlinDebug { CREATED_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeOrAbsolute(projectCacheDirProvider) }
} else {
log.kotlinDebug { EXISTING_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeOrAbsolute(projectCacheDirProvider) }
internal fun getOrCreateSessionFlagFile(
log: Logger,
sessionsDir: File,
): File {
sessionFileLock.read {
val sessionFlagRead = sessionFlagFile
if (sessionFlagRead != null && sessionFlagRead.exists()) {
return sessionFlagRead.sessionFileFlagExists(log)
}
}
return sessionFlagFile!!
sessionFileLock.write {
val sessionFlagWrite = sessionFlagFile
if (sessionFlagWrite != null && sessionFlagWrite.exists()) {
return sessionFlagWrite.sessionFileFlagExists(log)
}
return newTmpFile(
prefix = "kotlin-compiler-",
suffix = ".salive",
directory = sessionsDir.apply { mkdirs() })
.also {
sessionFlagFile = it
log.kotlinDebug { CREATED_SESSION_FILE_PREFIX + it.absolutePath }
}
}
}
internal fun sessionsDir(projectCacheDir: File): File =
File(File(projectCacheDir, "kotlin"), "sessions")
private fun File.sessionFileFlagExists(log: Logger): File {
log.kotlinDebug { EXISTING_SESSION_FILE_PREFIX + absolutePath }
return this
}
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt
+8 -2
View File
@@ -42,10 +42,16 @@ internal class ProjectFilesForCompilation(
val sessionFlagFile: File,
val buildDir: File
) : Serializable {
constructor(logger: Logger, projectDir:File, buildDir: File, projectName: String, projectCacheDirProvider: File, sessionDir: File) : this(
constructor(
logger: Logger,
projectDir: File,
buildDir: File,
projectName: String,
sessionDir: File
) : this(
projectRootFile = projectDir,
clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(logger, projectName),
sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir, projectCacheDirProvider),
sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(logger, sessionDir),
buildDir = buildDir
)
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleBuildServices.kt
+5 -4
View File
@@ -8,17 +8,18 @@ package org.jetbrains.kotlin.gradle.plugin
import org.gradle.api.Project
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.jetbrains.kotlin.gradle.logging.kotlinDebug
import org.jetbrains.kotlin.gradle.utils.projectCacheDir
import org.jetbrains.kotlin.gradle.utils.kotlinSessionsDir
import java.io.File
internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBuildServices.Parameters>, AutoCloseable {
interface Parameters : BuildServiceParameters {
var projectCacheDir: File
val sessionsDir: Property<File>
}
private val log = Logging.getLogger(this.javaClass)
@@ -61,7 +62,7 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
}
override fun close() {
buildHandler.buildFinished(parameters.projectCacheDir)
buildHandler.buildFinished(parameters.sessionsDir.get())
log.kotlinDebug(DISPOSE_MESSAGE)
}
@@ -75,7 +76,7 @@ internal abstract class KotlinGradleBuildServices : BuildService<KotlinGradleBui
"kotlin-build-service-${KotlinGradleBuildServices::class.java.canonicalName}_${KotlinGradleBuildServices::class.java.classLoader.hashCode()}",
KotlinGradleBuildServices::class.java
) { service ->
service.parameters.projectCacheDir = gradle.projectCacheDir
service.parameters.sessionsDir.set(gradle.rootProject.kotlinSessionsDir)
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinGradleFinishBuildHandler.kt
+3 -4
View File
@@ -32,24 +32,23 @@ internal class KotlinGradleFinishBuildHandler {
startMemory = getUsedMemoryKb()
}
fun buildFinished(projectCacheDir: File) {
fun buildFinished(sessionsDir: File) {
TaskLoggers.clear()
TaskExecutionResults.clear()
GradleCompilerRunner.clearBuildModulesInfo()
val sessionsDir = GradleCompilerRunner.sessionsDir(projectCacheDir)
if (sessionsDir.exists()) {
val sessionFiles = sessionsDir.listFiles()
// it is expected that only one session file per build exists
// afaik is is not possible to run multiple gradle builds in one project since gradle locks some dirs
if (sessionFiles.size > 1) {
log.warn("w: Detected multiple Kotlin daemon sessions at ${sessionsDir.relativeOrAbsolute(projectCacheDir)}")
log.warn("w: Detected multiple Kotlin daemon sessions at ${sessionsDir.relativeOrAbsolute(sessionsDir)}")
}
for (file in sessionFiles) {
file.delete()
log.kotlinDebug { DELETED_SESSION_FILE_PREFIX + file.relativeOrAbsolute(projectCacheDir) }
log.kotlinDebug { DELETED_SESSION_FILE_PREFIX + file.absolutePath }
}
}
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt
+2 -3
View File
@@ -543,10 +543,9 @@ internal class PropertiesProvider private constructor(private val project: Proje
* This data is not cleaned by gradle clean task. This dir can be used to store data
* for IDE import i.e. `./gradlew clean` will not break IDE import state.
*
* If the property is not set, the default value `$ROOT_DIR/.kotlin` is used.
* If the property is not set, the plugin will use `<user_home>/.kotlin` as default.
*/
val kotlinPersistentGradleDataDir: String
get() = get(PropertyNames.KOTLIN_PERSISTENT_GRADLE_DATA_DIR) ?: ".kotlin"
val kotlinPersistentGradleDataDir: String? = get(PropertyNames.KOTLIN_PERSISTENT_GRADLE_DATA_DIR)
/**
* Retrieves a comma-separated list of browsers to use when running karma tests for [target]
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt
+6 -3
View File
@@ -18,9 +18,12 @@ private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransforme
internal fun ProjectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
buildDirectory.get().asFile.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)
internal val Project.kotlinGradleDataDirCompat
get() = kotlinPropertiesProvider.kotlinPersistentGradleDataDir ?: ".kotlin"
internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File
get() = compositeBuildRootProject.rootDir
.resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
.resolve(project.kotlinGradleDataDirCompat)
.resolve(kotlinTransformedMetadataLibraries)
internal fun ProjectLayout.kotlinTransformedCInteropMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
@@ -28,10 +31,10 @@ internal fun ProjectLayout.kotlinTransformedCInteropMetadataLibraryDirectoryForB
internal val Project.kotlinCInteropLibraryDirectoryForIde: File
get() = compositeBuildRootProject.rootDir
.resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
.resolve(project.kotlinGradleDataDirCompat)
.resolve(kotlinCInteropLibraries)
internal val Project.kotlinTransformedCInteropMetadataLibraryDirectoryForIde: File
get() = compositeBuildRootProject.rootDir
.resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
.resolve(project.kotlinGradleDataDirCompat)
.resolve(kotlinTransformedCInteropMetadataLibraries)
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CopyCInteropCommonizerTaskOutputForIdeTask.kt
+2 -2
View File
@@ -16,7 +16,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
import org.jetbrains.kotlin.compilerRunner.addBuildMetricsForTaskAction
import org.jetbrains.kotlin.gradle.report.GradleBuildMetricsReporter
import org.jetbrains.kotlin.gradle.utils.property
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
import org.jetbrains.kotlin.gradle.plugin.mpp.kotlinGradleDataDirCompat
import java.io.File
import javax.inject.Inject
@@ -34,7 +34,7 @@ internal abstract class CopyCommonizeCInteropForIdeTask @Inject constructor(
@get:OutputDirectory
override val outputDirectory: File = project.rootDir
.resolve(project.kotlinPropertiesProvider.kotlinPersistentGradleDataDir)
.resolve(project.kotlinGradleDataDirCompat)
.resolve("commonizer")
.resolve(project.path.removePrefix(":").replace(":", "/"))
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/GradleCompileTaskProvider.kt
+2 -7
View File
@@ -14,11 +14,10 @@ import org.gradle.api.logging.Logger
import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner.Companion.normalizeForFlagFile
import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoProvider
import org.jetbrains.kotlin.gradle.utils.projectCacheDir
import org.jetbrains.kotlin.gradle.utils.property
import org.jetbrains.kotlin.gradle.utils.kotlinSessionsDir
import java.io.File
import javax.inject.Inject
@@ -44,13 +43,9 @@ abstract class GradleCompileTaskProvider @Inject constructor(
val projectDir: Provider<File> = objectFactory
.property(project.rootProject.projectDir)
@get:Internal
val projectCacheDir: Provider<File> = objectFactory
.property(gradle.projectCacheDir)
@get:Internal
val sessionsDir: Provider<File> = objectFactory
.property(GradleCompilerRunner.sessionsDir(gradle.projectCacheDir))
.property(project.kotlinSessionsDir)
@get:Internal
val projectName: Provider<String> = objectFactory
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/gradleUtils.kt
-3
View File
@@ -18,9 +18,6 @@ import org.gradle.api.services.BuildServiceSpec
import org.gradle.util.GradleVersion
import kotlin.reflect.KClass
val Gradle.projectCacheDir
get() = startParameter.projectCacheDir ?: this.rootProject.projectDir.resolve(".gradle")
internal val Project.compositeBuildRootGradle: Gradle get() = generateSequence(project.gradle) { it.parent }.last()
internal val Project.compositeBuildRootProject: Project get() = compositeBuildRootGradle.rootProject
libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/persistentCaches.kt
+28
View File
@@ -0,0 +1,28 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.utils
import org.gradle.api.Project
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
import java.io.File
import java.security.MessageDigest
private const val PROJECTS_CACHE_NAME = "projects"
private const val PROJECTS_CACHE_VERSION = 1
private const val PROJECTS_CACHE_NAME_FULL = "$PROJECTS_CACHE_NAME-$PROJECTS_CACHE_VERSION"
private const val SESSIONS_DIR_NAME = "sessions"
internal val Project.basePersistentDir
get() = kotlinPropertiesProvider.kotlinPersistentGradleDataDir?.let { File(it) }
?: File(System.getProperty("user.home")).resolve(".kotlin")
internal val Project.kotlinSessionsDir
get() = basePersistentDir.resolve(PROJECTS_CACHE_NAME_FULL).resolve(rootDir.absolutePathMd5Hash()).resolve(SESSIONS_DIR_NAME)
private val md5Digest by lazy { MessageDigest.getInstance("MD5") }
private fun File.absolutePathMd5Hash(): String = md5Digest.digest(absolutePath.toByteArray()).toHexString()