From 894ba9ab809c400de048d43fa98f89087100fcbc Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Thu, 10 Aug 2023 11:05:55 +0100 Subject: [PATCH] [IC] Relocatable IC caches for projects with custom buildDir IC caches often contain file paths. To make them relocatable, we need to convert these file paths into relative paths, relative to a base directory. - If the file paths are source files, we can use the root project directory as base. - If the file paths are class files, we should use the classes directory as base (before this commit, we used the root project directory in both cases, that's why we hit KT-58547). The key changes in this commit include: - RelocatableFileToPathConverter: converts paths to relative paths - IncrementalCompilationContext: contains 2 different path converters, one for source files and one for class files - SourceToOutputFilesMap: maps source files to class files using the above path converters - IncrementalCompilerRunner: creates the path converters based on file locations Test: RelocatableFileToPathConverterTest unit test SourceToOutputFilesMapTest unit test BuildCacheRelocationIT.testCustomBuildDirectory integration test ^KT-58547 Fixed --- .idea/dictionaries/hungnv.xml | 1 + .../IncrementalCompilationContext.kt | 27 +-- .../storage/FileToPathConverter.kt | 1 + .../storage/IncrementalFileToPathConverter.kt | 34 ---- .../kotlin/incremental/storage/LazyStorage.kt | 6 +- .../storage/RelocatableFileToPathConverter.kt | 54 ++++++ .../storage/InMemoryStorageWrapperTest.kt | 2 +- .../IncrementalFileToPathConverterTest.kt | 62 ------- .../storage/RelocatableCachesTest.kt | 4 +- .../RelocatableFileToPathConverterTest.kt | 38 +++++ .../api/kotlin-build-tools-api.api | 9 +- .../api/jvm/JvmCompilationConfiguration.kt | 42 +++-- .../JvmCompilationConfigurationImpl.kt | 21 ++- .../buildtools/internal/daemonAdapters.kt | 3 +- .../daemon/common/CompilationOptions.kt | 7 +- .../kotlin/daemon/CompileServiceImpl.kt | 17 +- .../incremental/IncrementalCompilerRunner.kt | 37 ++-- .../kotlin/incremental/InputsCache.kt | 4 +- .../classpathDiff/ClasspathChangesComputer.kt | 8 +- .../storage/SourceToOutputFilesMap.kt | 10 +- .../incremental/BuildDiffsStorageTest.kt | 2 +- .../snapshots/FileSnapshotMapTest.kt | 7 +- .../storage/SourceToOutputFilesMapTest.kt | 160 ++++++++---------- .../kotlin/jps/build/KotlinCompileContext.kt | 5 +- .../kotlin/gradle/BuildCacheRelocationIT.kt | 29 +++- .../GradleKotlinCompilerWork.kt | 9 +- .../IncrementalCompilationEnvironment.kt | 1 + .../btapi/BuildToolsApiCompilationWork.kt | 3 +- .../kotlin/gradle/tasks/Kotlin2JsCompile.kt | 10 +- .../kotlin/gradle/tasks/KotlinCompile.kt | 9 +- 30 files changed, 329 insertions(+), 293 deletions(-) delete mode 100644 build-common/src/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverter.kt create mode 100644 build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt delete mode 100644 build-common/test/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverterTest.kt create mode 100644 build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverterTest.kt diff --git a/.idea/dictionaries/hungnv.xml b/.idea/dictionaries/hungnv.xml index de66416ff33..5d413eb9296 100644 --- a/.idea/dictionaries/hungnv.xml +++ b/.idea/dictionaries/hungnv.xml @@ -4,6 +4,7 @@ externalizers granularities multifile + relocatability shrinker snapshotter snapshotter's diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt index ed8b9d16955..4ed9be33790 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt @@ -7,14 +7,13 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.build.report.DoNothingICReporter import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter import org.jetbrains.kotlin.incremental.storage.FileToPathConverter -import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter -import java.io.File - -private fun createDefaultPathConverter(rootProjectDir: File?) = IncrementalFileToPathConverter(rootProjectDir) class IncrementalCompilationContext( - val pathConverter: FileToPathConverter, + // The root directories of source files and class files are different, so we may need different `FileToPathConverter`s + val pathConverterForSourceFiles: FileToPathConverter = FileToAbsolutePathConverter, + val pathConverterForOutputFiles: FileToPathConverter = FileToAbsolutePathConverter, val storeFullFqNamesInLookupCache: Boolean = false, val transaction: CompilationTransaction = NonRecoverableCompilationTransaction(), val reporter: ICReporter = DoNothingICReporter, @@ -29,19 +28,7 @@ class IncrementalCompilationContext( */ val keepIncrementalCompilationCachesInMemory: Boolean = false, ) { - constructor( - rootProjectDir: File?, - storeFullFqNamesInLookupCache: Boolean = false, - transaction: CompilationTransaction = NonRecoverableCompilationTransaction(), - reporter: ICReporter = DoNothingICReporter, - trackChangesInLookupCache: Boolean = false, - keepIncrementalCompilationCachesInMemory: Boolean = false, - ) : this( - createDefaultPathConverter(rootProjectDir), - storeFullFqNamesInLookupCache, - transaction, - reporter, - trackChangesInLookupCache, - keepIncrementalCompilationCachesInMemory - ) + // FIXME: Remove `pathConverter` and require its users to decide whether to use `pathConverterForSourceFiles` or + // `pathConverterForClassFiles` + val pathConverter = pathConverterForSourceFiles } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt index d238776a9e7..d32708ca940 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.incremental.storage import java.io.File +/** Converts a [File] to a path of type [String] to store in IC caches, and vice versa. */ interface FileToPathConverter { fun toPath(file: File): String fun toFile(path: String): File diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverter.kt deleted file mode 100644 index 8f4441bbf00..00000000000 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverter.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.incremental.storage - -import java.io.File - -open class IncrementalFileToPathConverter(val rootProjectDir: File?) : FileToPathConverter { - //project root dir - private val projectDirPath = rootProjectDir?.normalize()?.absolutePath - - override fun toPath(file: File): String { - val path = file.normalize().absolutePath - return when { - projectDirPath == null || !path.startsWith(projectDirPath) -> path - else -> PROJECT_DIR_PLACEHOLDER + path.substring(projectDirPath.length) - } - } - - override fun toFile(path: String): File = - when { - rootProjectDir != null && path.startsWith(PROJECT_DIR_PLACEHOLDER) -> rootProjectDir.resolve(path.substring(PATH_PREFIX.length)) - else -> File(path) - } - - private companion object { - private const val PROJECT_DIR_PLACEHOLDER = "${'$'}PROJECT_DIR$" - //use only for prefix length because it OS dependent - private const val PATH_PREFIX = "$PROJECT_DIR_PLACEHOLDER/" - - } -} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt index 32dc81fa1c7..8452ee406f2 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt @@ -16,7 +16,9 @@ package org.jetbrains.kotlin.incremental.storage -interface LazyStorage { +import java.io.Closeable + +interface LazyStorage : Closeable { val keys: Collection operator fun contains(key: K): Boolean operator fun get(key: K): V? @@ -24,7 +26,7 @@ interface LazyStorage { fun remove(key: K) fun clean() fun flush(memoryCachesOnly: Boolean) - fun close() + override fun close() } interface AppendableLazyStorage : LazyStorage { diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt new file mode 100644 index 00000000000..12df621583f --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage + +import java.io.File + +/** + * [FileToPathConverter] which converts a [File] to a Unix-style relative path, which is relative to a base directory ([baseDir]), and vice + * versa. + * + * This is to support build cache relocatability + * (https://docs.gradle.org/current/userguide/build_cache_concepts.html#relocatability). + */ +class RelocatableFileToPathConverter(private val baseDir: File) : FileToPathConverter { + + private val unixStyleBaseDirPathPrefix = "${baseDir.invariantSeparatorsPath}/" + + override fun toPath(file: File): String { + check(file.invariantSeparatorsPath.startsWith(unixStyleBaseDirPathPrefix)) { + "The given file '${file.path}' is located outside the base directory '${baseDir.path}'" + } + return file.relativeTo(baseDir).invariantSeparatorsPath + } + + override fun toFile(path: String): File { + // Note: The given path is Unix-style but baseDir could be Windows-style; call normalize() so that the style of the returned file's + // path is consistent with baseDir. + return baseDir.resolve(path).normalize() + } +} + +/** + * File locations to support build cache relocatability (see [RelocatableFileToPathConverter]). + * + * These are the root directories of + * - Source files + * - Output files, which include .class files and possibly additional output files such as `.java` stub files for KaptGenerateStubs tasks. + * + * Ideally, they should be the most specific root directories (e.g., `/path/to/MyApplication/app/src/main/kotlin` for source files and + * `/path/to/MyApplication/app/build/tmp/kotlin-classes/debug` for output .class files). However, for simplicity we are using the root + * project directory ([rootProjectDir]) for source files and ([buildDir]) for output files. + */ +class FileLocations( + val rootProjectDir: File, + val buildDir: File, +) { + + fun getRelocatablePathConverterForSourceFiles() = RelocatableFileToPathConverter(rootProjectDir) + + fun getRelocatablePathConverterForOutputFiles() = RelocatableFileToPathConverter(buildDir) +} diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/InMemoryStorageWrapperTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/InMemoryStorageWrapperTest.kt index 791f2f80608..a154ff83a2c 100644 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/InMemoryStorageWrapperTest.kt +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/InMemoryStorageWrapperTest.kt @@ -125,7 +125,7 @@ class InMemoryStorageWrapperTest { val storageFile = storageRoot.resolve("lookup").toFile() val fileToPathConverter = RelativeFileToPathConverter(storageFile) val icContext = IncrementalCompilationContext( - pathConverter = fileToPathConverter, + pathConverterForSourceFiles = fileToPathConverter, keepIncrementalCompilationCachesInMemory = useInMemoryWrapper ) icContext.transaction.runWithin { transaction -> diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverterTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverterTest.kt deleted file mode 100644 index cfe017e0426..00000000000 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/IncrementalFileToPathConverterTest.kt +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.incremental.storage - -import org.jetbrains.kotlin.TestWithWorkingDir -import org.junit.Test -import java.io.File - -internal class IncrementalFileToPathConverterTest : TestWithWorkingDir() { - val separator: String = File.separator - - @Test - fun testPathTransform() { - val relativeFilePath = "testFile.txt" - val transformedPath = testPathTransformation(workingDir.resolve("testDir"), relativeFilePath) - - assertEquals("${'$'}PROJECT_DIR${'$'}$separator$relativeFilePath", transformedPath) - } - - @Test - fun testComplicatedProjectRootPath() { - val relativeFilePath = "testFile.txt" - val transformedPath = testPathTransformation(workingDir.resolve("first$separator..${separator}testDir"), relativeFilePath) - - assertEquals("${'$'}PROJECT_DIR${'$'}$separator$relativeFilePath", transformedPath) - } - - @Test - fun testInccorectProjectRootPath() { - val relativeFilePath = "testFile.txt" - val transformedPath = testPathTransformation(workingDir.resolve("testDir$separator"), relativeFilePath) - - assertEquals("${'$'}PROJECT_DIR${'$'}$separator$relativeFilePath", transformedPath) - } - - @Test - fun testFileOutOfProject() { - val relativeFilePath = "..${separator}testFile.txt" - val transformedPath = testPathTransformation(workingDir.resolve("testDir"), relativeFilePath) - - assertEquals("${workingDir.absolutePath}${separator}testFile.txt", transformedPath) - } - - @Test - fun testFileWithExtraSlash() { - val relativeFilePath = "testFile.txt$separator" - val transformedPath = testPathTransformation(workingDir.resolve("testDir"), relativeFilePath) - - assertEquals("${'$'}PROJECT_DIR${'$'}${separator}testFile.txt", transformedPath) - } - - private fun testPathTransformation(projectRoot: File, relativeFilePath: String): String { - val pathConverter = IncrementalFileToPathConverter(projectRoot) - val testFile = projectRoot.resolve(relativeFilePath) - val transformedPath = pathConverter.toPath(testFile) - assertEquals(testFile.normalize().absolutePath, pathConverter.toFile(transformedPath).normalize().absolutePath) - return transformedPath - } -} \ No newline at end of file diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt index 15e382687ae..e09ce6734db 100644 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt @@ -7,8 +7,6 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.containers.MultiMap import org.jetbrains.kotlin.TestWithWorkingDir -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties -import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.LookupStorage import org.jetbrains.kotlin.incremental.LookupSymbol @@ -52,7 +50,7 @@ class RelocatableCachesTest : TestWithWorkingDir() { val storageRoot = projectRoot.storageRoot val fileToPathConverter = RelativeFileToPathConverter(projectRoot) val icContext = IncrementalCompilationContext( - pathConverter = fileToPathConverter, + pathConverterForSourceFiles = fileToPathConverter, storeFullFqNamesInLookupCache = storeFullFqNames, ) val lookupStorage = LookupStorage(storageRoot, icContext) diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverterTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverterTest.kt new file mode 100644 index 00000000000..461d28effa8 --- /dev/null +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverterTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage + +import org.jetbrains.kotlin.TestWithWorkingDir +import org.junit.Test +import kotlin.test.assertFailsWith + +class RelocatableFileToPathConverterTest : TestWithWorkingDir() { + + private lateinit var pathConverter: RelocatableFileToPathConverter + + override fun setUp() { + super.setUp() + pathConverter = RelocatableFileToPathConverter(workingDir) + } + + @Test + fun testToPath() { + assertEquals("com/example/Foo.kt", pathConverter.toPath(workingDir.resolve("com/example/Foo.kt"))) + } + + @Test + fun testToPathFails() { + assertFailsWith(IllegalStateException::class) { + pathConverter.toPath(workingDir.resolve("../outsideWorkingDir/com/example/Foo.kt").normalize()) + } + } + + @Test + fun testToFile() { + assertEquals(workingDir.resolve("com/example/Foo.kt"), pathConverter.toFile("com/example/Foo.kt")) + } + +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api index 391359f453d..3df6793b7df 100644 --- a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api +++ b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api @@ -113,10 +113,11 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Classpat public abstract fun forceNonIncrementalMode (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun getAssuredNoClasspathSnapshotsChanges ()Z public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; + public abstract fun setBuildDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; + public abstract fun setRootProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; - public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; } public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/InaccessibleClassSnapshot : org/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshot { @@ -128,17 +129,19 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Incremen public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration { public abstract fun forceNonIncrementalMode (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public static synthetic fun forceNonIncrementalMode$default (Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration;ZILjava/lang/Object;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; + public abstract fun getBuildDir ()Ljava/io/File; public abstract fun getForcedNonIncrementalMode ()Z public abstract fun getIncrementalCompilationCachesKeptInMemory ()Z public abstract fun getOutputDirs ()Ljava/util/Set; public abstract fun getPreciseCompilationResultsBackupEnabled ()Z public abstract fun getPreciseJavaTrackingEnabled ()Z - public abstract fun getProjectDir ()Ljava/io/File; + public abstract fun getRootProjectDir ()Ljava/io/File; public abstract fun keepIncrementalCompilationCachesInMemory (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; + public abstract fun setBuildDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; + public abstract fun setRootProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun useOutputDirs (Ljava/util/Collection;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun usePreciseCompilationResultsBackup (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; public abstract fun usePreciseJavaTracking (Z)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; - public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/IncrementalJvmCompilationConfiguration; } public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration { diff --git a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt index 3a490ee3754..74c65b19f74 100644 --- a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt +++ b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration.kt @@ -78,20 +78,36 @@ public interface JvmCompilationConfiguration { */ @ExperimentalBuildToolsApi public interface IncrementalJvmCompilationConfiguration

{ - /** - * A directory used as a base path for computing relative paths in the incremental compilation caches. - * - * If is not specified, incremental compilation caches will be non-relocatable - * - * Managed by [useProjectDir] - * Default values is `null` - */ - public val projectDir: File? /** - * @see [IncrementalJvmCompilationConfiguration.projectDir] + * The root project directory, used for computing relative paths in the incremental compilation caches. + * + * If it is not specified, incremental compilation caches will be non-relocatable. + * + * Managed by [setRootProjectDir] + * Default value is `null` */ - public fun useProjectDir(projectDir: File): IncrementalJvmCompilationConfiguration

+ public val rootProjectDir: File? + + /** + * @see [IncrementalJvmCompilationConfiguration.rootProjectDir] + */ + public fun setRootProjectDir(rootProjectDir: File): IncrementalJvmCompilationConfiguration

+ + /** + * The build directory, used for computing relative paths in the incremental compilation caches. + * + * If it is not specified, incremental compilation caches will be non-relocatable. + * + * Managed by [setBuildDir] + * Default value is `null` + */ + public val buildDir: File? + + /** + * @see [IncrementalJvmCompilationConfiguration.buildDir] + */ + public fun setBuildDir(buildDir: File): IncrementalJvmCompilationConfiguration

/** * An indicator whether incremental compilation will analyze Java files precisely for better changes detection @@ -186,7 +202,9 @@ public interface ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration : */ public fun assureNoClasspathSnapshotsChanges(value: Boolean = true): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration - override fun useProjectDir(projectDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration + override fun setRootProjectDir(rootProjectDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration + + override fun setBuildDir(buildDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration override fun usePreciseJavaTracking(value: Boolean): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt index 204ea811524..7e11946be4f 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/JvmCompilationConfigurationImpl.kt @@ -50,12 +50,18 @@ internal abstract class JvmIncrementalCompilationConfigurationImpl

= emptySet(), ) : IncrementalJvmCompilationConfiguration

{ - override fun useProjectDir(projectDir: File): IncrementalJvmCompilationConfiguration

{ - this.projectDir = projectDir + override fun setRootProjectDir(rootProjectDir: File): IncrementalJvmCompilationConfiguration

{ + this.rootProjectDir = rootProjectDir + return this + } + + override fun setBuildDir(buildDir: File): IncrementalJvmCompilationConfiguration

{ + this.buildDir = buildDir return this } @@ -90,8 +96,13 @@ internal class ClasspathSnapshotBasedIncrementalJvmCompilationConfigurationImpl( ) : JvmIncrementalCompilationConfigurationImpl(), ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration { - override fun useProjectDir(projectDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration { - super.useProjectDir(projectDir) + override fun setRootProjectDir(rootProjectDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration { + super.setRootProjectDir(rootProjectDir) + return this + } + + override fun setBuildDir(buildDir: File): ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration { + super.setBuildDir(buildDir) return this } diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt index 1f554cee089..2618557d36e 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/daemonAdapters.kt @@ -56,7 +56,8 @@ internal val JvmCompilationConfigurationImpl.asDaemonCompilationOptions: Compila outputFiles = options.outputDirs, multiModuleICSettings = null, // required only for the build history approach modulesInfo = null, // required only for the build history approach - rootProjectDir = options.projectDir, + rootProjectDir = options.rootProjectDir, + buildDir = options.buildDir, kotlinScriptExtensions = ktsExtensionsAsArray, withAbiSnapshot = false, preciseCompilationResultsBackup = options.preciseCompilationResultsBackupEnabled, diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt index 95e1e7cdfbd..cbc830835e9 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt @@ -70,10 +70,11 @@ class IncrementalCompilationOptions( val outputFiles: Collection? = null, val multiModuleICSettings: MultiModuleICSettings? = null, val modulesInfo: IncrementalModuleInfo? = null, - /** - * Root project directory, used to resolve relative paths - */ + + // rootProjectDir and buildDir are used to resolve relative paths val rootProjectDir: File?, + val buildDir: File?, + kotlinScriptExtensions: Array? = null, val withAbiSnapshot: Boolean = false, val preciseCompilationResultsBackup: Boolean = false, diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index d12117536f7..9e31970eb2c 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -60,6 +60,7 @@ import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJvm import org.jetbrains.kotlin.incremental.parsing.classesFqNames +import org.jetbrains.kotlin.incremental.storage.FileLocations import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents import org.jetbrains.kotlin.progress.CompilationCanceledStatus import java.io.File @@ -612,20 +613,21 @@ abstract class CompileServiceImplBase( val workingDir = incrementalCompilationOptions.workingDir - val projectRoot = incrementalCompilationOptions.rootProjectDir + val rootProjectDir = incrementalCompilationOptions.rootProjectDir + val buildDir = incrementalCompilationOptions.buildDir val modulesApiHistory = incrementalCompilationOptions.multiModuleICSettings?.run { reporter.info { "Use module detection: $useModuleDetection" } val modulesInfo = incrementalCompilationOptions.modulesInfo ?: error("The build is configured to use the history-file based IC approach, but doesn't provide the modulesInfo") - check(projectRoot != null) { + check(rootProjectDir != null) { "rootProjectDir is expected to be non null when the history-file based IC approach is used" } if (!useModuleDetection) { - ModulesApiHistoryJvm(projectRoot, modulesInfo) + ModulesApiHistoryJvm(rootProjectDir, modulesInfo) } else { - ModulesApiHistoryAndroid(projectRoot, modulesInfo) + ModulesApiHistoryAndroid(rootProjectDir, modulesInfo) } } ?: EmptyModulesApiHistory @@ -648,7 +650,12 @@ abstract class CompileServiceImplBase( keepIncrementalCompilationCachesInMemory = incrementalCompilationOptions.keepIncrementalCompilationCachesInMemory, ) return try { - compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, projectRoot) + compiler.compile( + allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles, + fileLocations = if (rootProjectDir != null && buildDir != null) { + FileLocations(rootProjectDir, buildDir) + } else null + ) } finally { reporter.endMeasureGc() reporter.flush() diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 2c75cecc090..b103f91d4f3 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -21,8 +21,11 @@ import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.report.BuildReporter import org.jetbrains.kotlin.build.report.debug import org.jetbrains.kotlin.build.report.info -import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.BuildAttribute.* +import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.build.report.warn import org.jetbrains.kotlin.cli.common.* import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments @@ -35,6 +38,8 @@ import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.parsing.classesFqNames +import org.jetbrains.kotlin.incremental.storage.FileLocations +import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector import org.jetbrains.kotlin.incremental.util.ExceptionLocation import org.jetbrains.kotlin.incremental.util.reportException @@ -45,7 +50,6 @@ import org.jetbrains.kotlin.util.removeSuffixIfPresent import org.jetbrains.kotlin.utils.toMetadataVersion import java.io.File import java.nio.file.Files -import java.util.* abstract class IncrementalCompilerRunner< Args : CommonCompilerArguments, @@ -83,11 +87,12 @@ abstract class IncrementalCompilerRunner< * Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager] */ private fun createIncrementalCompilationContext( - projectDir: File?, - transaction: CompilationTransaction + fileLocations: FileLocations?, + transaction: CompilationTransaction, ) = IncrementalCompilationContext( + pathConverterForSourceFiles = fileLocations?.let { it.getRelocatablePathConverterForSourceFiles() } ?: FileToAbsolutePathConverter, + pathConverterForOutputFiles = fileLocations?.let { it.getRelocatablePathConverterForOutputFiles() } ?: FileToAbsolutePathConverter, transaction = transaction, - rootProjectDir = projectDir, reporter = reporter, trackChangesInLookupCache = shouldTrackChangesInLookupCache, storeFullFqNamesInLookupCache = shouldStoreFullFqNamesInLookupCache, @@ -105,12 +110,10 @@ abstract class IncrementalCompilerRunner< allSourceFiles: List, args: Args, messageCollector: MessageCollector, - // when `changedFiles` is not null, changes are provided by external system (e.g. Gradle) - // otherwise we track source files changes ourselves. - changedFiles: ChangedFiles?, - projectDir: File? = null + changedFiles: ChangedFiles?, // When not-null, changes are provided by the build system; otherwise, the IC will need to track them + fileLocations: FileLocations? = null, // Must be not-null if the build system needs to support build cache relocatability ): ExitCode = reporter.measure(GradleBuildTime.INCREMENTAL_COMPILATION_DAEMON) { - return when (val result = tryCompileIncrementally(allSourceFiles, changedFiles, args, projectDir, messageCollector)) { + return when (val result = tryCompileIncrementally(allSourceFiles, changedFiles, args, fileLocations, messageCollector)) { is ICResult.Completed -> { reporter.debug { "Incremental compilation completed" } result.exitCode @@ -120,7 +123,7 @@ abstract class IncrementalCompilerRunner< reporter.addAttribute(result.reason) compileNonIncrementally( - result.reason, allSourceFiles, args, projectDir, trackChangedFiles = changedFiles == null, messageCollector + result.reason, allSourceFiles, args, fileLocations, trackChangedFiles = changedFiles == null, messageCollector ) } is ICResult.Failed -> { @@ -139,7 +142,7 @@ abstract class IncrementalCompilerRunner< reporter.addAttribute(result.reason) compileNonIncrementally( - result.reason, allSourceFiles, args, projectDir, trackChangedFiles = changedFiles == null, messageCollector + result.reason, allSourceFiles, args, fileLocations, trackChangedFiles = changedFiles == null, messageCollector ) } } @@ -173,8 +176,8 @@ abstract class IncrementalCompilerRunner< allSourceFiles: List, changedFiles: ChangedFiles?, args: Args, - projectDir: File?, - messageCollector: MessageCollector + fileLocations: FileLocations?, + messageCollector: MessageCollector, ): ICResult { if (changedFiles is ChangedFiles.Unknown) { return ICResult.RequiresRebuild(UNKNOWN_CHANGES_IN_GRADLE_INPUTS) @@ -182,7 +185,7 @@ abstract class IncrementalCompilerRunner< changedFiles as ChangedFiles.Known? return createTransaction().runWithin(::incrementalCompilationExceptionTransformer) { transaction -> - val icContext = createIncrementalCompilationContext(projectDir, transaction) + val icContext = createIncrementalCompilationContext(fileLocations, transaction) val caches = createCacheManager(icContext, args).also { // this way we make the transaction to be responsible for closing the caches manager transaction.cachesManager = it @@ -253,7 +256,7 @@ abstract class IncrementalCompilerRunner< rebuildReason: BuildAttribute, allSourceFiles: List, args: Args, - projectDir: File?, + fileLocations: FileLocations?, trackChangedFiles: Boolean, // Whether we need to track changes to the source files or the build system already handles it messageCollector: MessageCollector, ): ExitCode { @@ -266,7 +269,7 @@ abstract class IncrementalCompilerRunner< reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" } cleanOrCreateDirectories(outputDirsToClean) } - val icContext = createIncrementalCompilationContext(projectDir, NonRecoverableCompilationTransaction()) + val icContext = createIncrementalCompilationContext(fileLocations, NonRecoverableCompilationTransaction()) return createCacheManager(icContext, args).use { caches -> if (trackChangedFiles) { caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt index 5a5a9267c32..c3af3c2bbc6 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt @@ -38,7 +38,7 @@ class InputsCache( fun removeOutputForSourceFiles(sources: Iterable) { for (sourceFile in sources) { - sourceToOutputMap.remove(sourceFile).forEach { + sourceToOutputMap.getAndRemove(sourceFile)?.forEach { icContext.reporter.debug { "Deleting $it on clearing cache for $sourceFile" } icContext.transaction.deleteFile(it.toPath()) } @@ -46,7 +46,7 @@ class InputsCache( } fun getOutputForSourceFiles(sources: Iterable): List = sources.flatMap { - sourceToOutputMap[it] + sourceToOutputMap[it].orEmpty() } // generatedFiles can contain multiple entries with the same source file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt index 68cfdda530e..6281a6bc6f1 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt @@ -8,12 +8,14 @@ package org.jetbrains.kotlin.incremental.classpathDiff import com.intellij.openapi.util.io.FileUtil import org.jetbrains.kotlin.build.report.DoNothingICReporter import org.jetbrains.kotlin.build.report.debug -import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.classpathDiff.BreadthFirstSearch.findReachableNodes import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrinkClasspath import org.jetbrains.kotlin.incremental.classpathDiff.ImpactedSymbolsComputer.computeImpactedSymbols -import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter import org.jetbrains.kotlin.incremental.storage.ListExternalizer import org.jetbrains.kotlin.incremental.storage.loadFromFile import org.jetbrains.kotlin.name.ClassId @@ -202,7 +204,7 @@ object ClasspathChangesComputer { ): ProgramSymbolSet { val workingDir = FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true) - val icContext = IncrementalCompilationContext(pathConverter = FileToAbsolutePathConverter) + val icContext = IncrementalCompilationContext() val incrementalJvmCache = IncrementalJvmCache(workingDir, icContext, null) // Step 1: diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt index 321aebcd29a..1089582a27b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt @@ -14,15 +14,15 @@ class SourceToOutputFilesMap( icContext: IncrementalCompilationContext, ) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { operator fun set(sourceFile: File, outputFiles: Collection) { - storage[pathConverter.toPath(sourceFile)] = outputFiles.map(pathConverter::toPath) + storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] = outputFiles.map(icContext.pathConverterForOutputFiles::toPath) } - operator fun get(sourceFile: File): Collection = - storage[pathConverter.toPath(sourceFile)].orEmpty().map(pathConverter::toFile) + operator fun get(sourceFile: File): Collection? = + storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)]?.map(icContext.pathConverterForOutputFiles::toFile) override fun dumpValue(value: Collection) = value.dumpCollection() - fun remove(file: File): Collection = - get(file).also { storage.remove(pathConverter.toPath(file)) } + fun getAndRemove(file: File): Collection? = + get(file).also { storage.remove(icContext.pathConverterForSourceFiles.toPath(file)) } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/BuildDiffsStorageTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/BuildDiffsStorageTest.kt index 7f2be9e9058..f633cbc0588 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/BuildDiffsStorageTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/BuildDiffsStorageTest.kt @@ -28,7 +28,7 @@ import java.util.* class BuildDiffsStorageTest { lateinit var storageFile: File private val random = Random(System.currentTimeMillis()) - private val icContext = IncrementalCompilationContext(null) + private val icContext = IncrementalCompilationContext() @Before fun setUp() { diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt index 0fdcb6ae005..2c2e0aabc74 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt @@ -7,8 +7,7 @@ package org.jetbrains.kotlin.incremental.snapshots import org.jetbrains.kotlin.TestWithWorkingDir import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.storage.FileToPathConverter -import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter +import org.jetbrains.kotlin.incremental.storage.RelocatableFileToPathConverter import org.junit.After import org.junit.Assert.assertArrayEquals import org.junit.Before @@ -24,9 +23,9 @@ class FileSnapshotMapTest : TestWithWorkingDir() { super.setUp() val caches = File(workingDir, "caches").apply { mkdirs() } val snapshotMapFile = File(caches, "snapshots.tab") - val pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile)) + val pathConverter = RelocatableFileToPathConverter((workingDir.canonicalFile)) val icContext = IncrementalCompilationContext( - pathConverter = pathConverter + pathConverterForSourceFiles = pathConverter ) snapshotMap = FileSnapshotMap(snapshotMapFile, icContext) } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt index 3b8ddf48a4d..ba25f45f97e 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt @@ -5,131 +5,105 @@ package org.jetbrains.kotlin.incremental.storage -import org.jetbrains.kotlin.TestWithWorkingDir +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNull import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.junit.After -import org.junit.Assert.assertArrayEquals import org.junit.Before +import org.junit.Rule import org.junit.Test +import org.junit.rules.TemporaryFolder import java.io.File -import kotlin.properties.Delegates +import kotlin.test.assertFailsWith -class SourceToOutputFilesMapTest : TestWithWorkingDir() { - private var stofMap: SourceToOutputFilesMap by Delegates.notNull() - private var pathConverter: FileToPathConverter by Delegates.notNull() +class SourceToOutputFilesMapTest { + + @get:Rule + val tmpDir = TemporaryFolder() + + private lateinit var srcDir: File + private lateinit var classesDir: File + + private lateinit var stofMap: SourceToOutputFilesMap + + private lateinit var fooDotKt: File + private lateinit var fooDotClass: File @Before - override fun setUp() { - super.setUp() - val caches = File(workingDir, "caches").apply { mkdirs() } - val stofMapFile = File(caches, "stof.tab") - pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile)) - val icContext = IncrementalCompilationContext( - pathConverter = pathConverter + fun setUp() { + val workingDir = tmpDir.root + + srcDir = workingDir.resolve("src") + classesDir = workingDir.resolve("classes") + + stofMap = SourceToOutputFilesMap( + storageFile = workingDir.resolve("stof.tab"), + icContext = IncrementalCompilationContext( + pathConverterForSourceFiles = RelocatableFileToPathConverter(srcDir), + pathConverterForOutputFiles = RelocatableFileToPathConverter(classesDir), + ) ) - stofMap = SourceToOutputFilesMap(stofMapFile, icContext) + + fooDotKt = srcDir.resolve("Foo.kt") + fooDotClass = classesDir.resolve("Foo.class") } @After - override fun tearDown() { + fun tearDown() { stofMap.flush(false) stofMap.closeForTest() - super.tearDown() } @Test - fun testEmptyGetReturnsEmpty() { - assertTrue(stofMap.get(File("")).isEmpty()) + fun testNoSetGetReturnsNull() { + assertNull(stofMap[fooDotKt]) } @Test - fun testSetGetOneReturnsOne() { - stofMap.set( - File(""), - listOf(File("one").canonicalFile)) - assertEquals( - listOf(File("one").canonicalFile), - stofMap.get(File(""))) + fun testSetOneGetReturnsOne() { + stofMap[fooDotKt] = listOf(fooDotClass) + + assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) } @Test - fun testSetDupeReturnsUnique() { - stofMap.set( - File(""), - listOf(File("one").canonicalFile, File("one").canonicalFile, File("one").canonicalFile)) + fun testSetDupeGetReturnsUnique() { + stofMap[fooDotKt] = listOf(fooDotClass, fooDotClass) - assertEquals( - listOf(File("one").canonicalFile), - stofMap.get(File(""))) + assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) } @Test - fun testSetOverwriteReturnsNew() { - stofMap.set( - File(""), - listOf(File("old").canonicalFile, File("old").canonicalFile, File("old").canonicalFile)) - stofMap.set( - File(""), - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile)) + fun testSetOverwriteGetReturnsNew() { + val fooKtDotClass = classesDir.resolve("FooKt.class") + stofMap[fooDotKt] = listOf(fooDotClass) + stofMap[fooDotKt] = listOf(fooKtDotClass) - assertArrayEquals( - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(), - stofMap.get(File("")).toSortedPaths()) + assertEquals(listOf(fooKtDotClass), stofMap[fooDotKt]) } @Test - fun testRelativeInReturnsAbsolute() { - stofMap.set( - File(""), - listOf(File("one"), File("two"), File("three"))) - - assertArrayEquals( - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(), - stofMap.get(File("")).toSortedPaths() - ) - } - - @Test - fun testSetRelativeGetAbsolute() { - stofMap.set( - File("blah"), - listOf(File("one"), File("two"), File("three"))) - - assertArrayEquals( - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(), - stofMap.get(File("blah").canonicalFile).toSortedPaths() - ) - } - - @Test - fun testSetRemove() { - stofMap.set( - File("blah"), - listOf(File("one"), File("two"), File("three"))) - - assertArrayEquals( - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(), - stofMap.remove(File("blah")).toSortedPaths() - ) - assertTrue(stofMap.get(File("blah")).isEmpty()) - } - - @Test - fun testSetRemoveLoop() { - repeat(5) { - stofMap.set( - File("blah"), - listOf(File("one"), File("two"), File("three")) - ) - - assertArrayEquals( - listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(), - stofMap.remove(File("blah")).toSortedPaths() - ) - assertTrue(stofMap.get(File("blah")).isEmpty()) + fun testSetRelativeFails() { + assertFailsWith { + stofMap[fooDotKt] = listOf(File("relativePath")) } } - private fun Iterable.toSortedPaths(): Array = - map { it.canonicalPath }.sorted().toTypedArray() + @Test + fun testGetRelativeFails() { + stofMap[fooDotKt] = listOf(fooDotClass) + + assertFailsWith { + stofMap[fooDotKt.relativeTo(srcDir)] + } + } + + @Test + fun testGetAndRemove() { + stofMap[fooDotKt] = listOf(fooDotClass) + + assertEquals(listOf(fooDotClass), stofMap.getAndRemove(fooDotKt)) + assertNull(stofMap[fooDotKt]) + } + } \ No newline at end of file diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt index 0b66c4d917d..9caaeddccc9 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt @@ -82,7 +82,10 @@ class KotlinCompileContext(val jpsContext: CompileContext) { val fileToPathConverter: FileToPathConverter = JpsFileToPathConverter(jpsContext.projectDescriptor.project) - val icContext = IncrementalCompilationContext(pathConverter = fileToPathConverter) + val icContext = IncrementalCompilationContext( + pathConverterForSourceFiles = fileToPathConverter, + pathConverterForOutputFiles = fileToPathConverter + ) val lookupStorageManager = JpsLookupStorageManager(dataManager, icContext) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt index db87e74587f..ac5967191e9 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildCacheRelocationIT.kt @@ -17,13 +17,15 @@ package org.jetbrains.kotlin.gradle import org.gradle.api.logging.LogLevel -import org.gradle.api.logging.configuration.WarningMode import org.gradle.testkit.runner.BuildResult import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.testbase.* import org.junit.jupiter.api.DisplayName import java.io.File import kotlin.io.path.createDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.readText +import kotlin.io.path.walk @DisplayName("Build cache relocation") class BuildCacheRelocationIT : KGPBaseTest() { @@ -400,4 +402,29 @@ class BuildCacheRelocationIT : KGPBaseTest() { assertTasksFromCache(":app:kaptGenerateStubsKotlin", ":app:kaptKotlin") } } + + @JvmGradlePluginTests + @DisplayName("test relocatability for projects using custom build directory") // Regression test for KT-58547 + @GradleTest + fun testCustomBuildDirectory(gradleVersion: GradleVersion) { + val (firstProject, secondProject) = prepareTestProjects("buildCacheSimple", gradleVersion) + firstProject.buildGradle.append("buildDir = \"../BUILD_DIR_1\"") + secondProject.buildGradle.append("buildDir = \"../BUILD_DIR_2\"") + + firstProject.build(":compileKotlin") + + val outputFilesContainingNonRelocatablePaths = + firstProject.projectPath.resolve("../BUILD_DIR_1/kotlin/compileKotlin").walk() + .filter { + // Use readText() even for binary files as we don't have a better way for now + it.isRegularFile() && it.readText().contains("BUILD_DIR_1") + }.toList() + assert(outputFilesContainingNonRelocatablePaths.isEmpty()) { + "The following output files contain non-relocatable paths:\n" + outputFilesContainingNonRelocatablePaths.joinToString("\n") + } + + secondProject.build(":compileKotlin") { + assertTasksFromCache(":compileKotlin") + } + } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index c52751ea9b1..5aefbe49284 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.logging.Logger import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.statistics.StatTag +import org.jetbrains.kotlin.buildtools.api.KotlinLogger import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.config.Services @@ -15,13 +17,8 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.logging.* import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers -import org.jetbrains.kotlin.build.report.statistics.StatTag -import org.jetbrains.kotlin.buildtools.api.KotlinLogger import org.jetbrains.kotlin.gradle.report.* import org.jetbrains.kotlin.gradle.tasks.* -import org.jetbrains.kotlin.gradle.tasks.OOMErrorException -import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState -import org.jetbrains.kotlin.gradle.tasks.kotlinDaemonOOMHelperMessage import org.jetbrains.kotlin.gradle.utils.stackTraceAsString import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ClasspathChanges @@ -38,7 +35,6 @@ import java.util.* import java.util.concurrent.Callable import java.util.concurrent.Executors import javax.inject.Inject -import kotlin.collections.HashSet internal class ProjectFilesForCompilation( val projectRootFile: File, @@ -331,6 +327,7 @@ internal class GradleKotlinCompilerWork @Inject constructor( multiModuleICSettings = icEnv.multiModuleICSettings, modulesInfo = incrementalModuleInfo!!, rootProjectDir = icEnv.rootProjectDir, + buildDir = icEnv.buildDir, kotlinScriptExtensions = kotlinScriptExtensions, withAbiSnapshot = icEnv.withAbiSnapshot, preciseCompilationResultsBackup = icEnv.preciseCompilationResultsBackup, diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt index 3ccf2e3842e..f3d855d4d15 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationEnvironment.kt @@ -16,6 +16,7 @@ internal class IncrementalCompilationEnvironment( val classpathChanges: ClasspathChanges, val workingDir: File, val rootProjectDir: File, + val buildDir: File, val usePreciseJavaTracking: Boolean = false, val disableMultiModuleIC: Boolean = false, val multiModuleICSettings: MultiModuleICSettings, diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt index 1fac1356cc1..41403fdac6a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt @@ -96,7 +96,8 @@ internal abstract class BuildToolsApiCompilationWork : WorkAction