[IC] Relocatable IC caches for projects with custom buildDir

IC caches often contain file paths. To make them relocatable, we need
to convert these file paths into relative paths, relative to a base
directory.
  - If the file paths are source files, we can use the root project
    directory as base.
  - If the file paths are class files, we should use the classes
    directory as base (before this commit, we used the root project
    directory in both cases, that's why we hit KT-58547).

The key changes in this commit include:
  - RelocatableFileToPathConverter: converts paths to relative paths
  - IncrementalCompilationContext: contains 2 different path converters,
    one for source files and one for class files
  - SourceToOutputFilesMap: maps source files to class files using the
    above path converters
  - IncrementalCompilerRunner: creates the path converters based on file
    locations

Test: RelocatableFileToPathConverterTest unit test
      SourceToOutputFilesMapTest unit test
      BuildCacheRelocationIT.testCustomBuildDirectory integration test
^KT-58547 Fixed

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -21,8 +21,11 @@ import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.report.BuildReporter
 import org.jetbrains.kotlin.build.report.debug
 import org.jetbrains.kotlin.build.report.info
-import org.jetbrains.kotlin.build.report.metrics.*
+import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
 import org.jetbrains.kotlin.build.report.metrics.BuildAttribute.*
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
+import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.build.report.warn
 import org.jetbrains.kotlin.cli.common.*
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
@@ -35,6 +38,8 @@ import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.parsing.classesFqNames
+import org.jetbrains.kotlin.incremental.storage.FileLocations
+import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter
 import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
 import org.jetbrains.kotlin.incremental.util.ExceptionLocation
 import org.jetbrains.kotlin.incremental.util.reportException
@@ -45,7 +50,6 @@ import org.jetbrains.kotlin.util.removeSuffixIfPresent
 import org.jetbrains.kotlin.utils.toMetadataVersion
 import java.io.File
 import java.nio.file.Files
-import java.util.*
 
 abstract class IncrementalCompilerRunner<
         Args : CommonCompilerArguments,
@@ -83,11 +87,12 @@ abstract class IncrementalCompilerRunner<
      * Creates an instance of [IncrementalCompilationContext] that holds common incremental compilation context mostly required for [CacheManager]
      */
     private fun createIncrementalCompilationContext(
-        projectDir: File?,
-        transaction: CompilationTransaction
+        fileLocations: FileLocations?,
+        transaction: CompilationTransaction,
     ) = IncrementalCompilationContext(
+        pathConverterForSourceFiles = fileLocations?.let { it.getRelocatablePathConverterForSourceFiles() } ?: FileToAbsolutePathConverter,
+        pathConverterForOutputFiles = fileLocations?.let { it.getRelocatablePathConverterForOutputFiles() } ?: FileToAbsolutePathConverter,
         transaction = transaction,
-        rootProjectDir = projectDir,
         reporter = reporter,
         trackChangesInLookupCache = shouldTrackChangesInLookupCache,
         storeFullFqNamesInLookupCache = shouldStoreFullFqNamesInLookupCache,
@@ -105,12 +110,10 @@ abstract class IncrementalCompilerRunner<
         allSourceFiles: List<File>,
         args: Args,
         messageCollector: MessageCollector,
-        // when `changedFiles` is not null, changes are provided by external system (e.g. Gradle)
-        // otherwise we track source files changes ourselves.
-        changedFiles: ChangedFiles?,
-        projectDir: File? = null
+        changedFiles: ChangedFiles?, // When not-null, changes are provided by the build system; otherwise, the IC will need to track them
+        fileLocations: FileLocations? = null, // Must be not-null if the build system needs to support build cache relocatability
     ): ExitCode = reporter.measure(GradleBuildTime.INCREMENTAL_COMPILATION_DAEMON) {
-        return when (val result = tryCompileIncrementally(allSourceFiles, changedFiles, args, projectDir, messageCollector)) {
+        return when (val result = tryCompileIncrementally(allSourceFiles, changedFiles, args, fileLocations, messageCollector)) {
             is ICResult.Completed -> {
                 reporter.debug { "Incremental compilation completed" }
                 result.exitCode
@@ -120,7 +123,7 @@ abstract class IncrementalCompilerRunner<
                 reporter.addAttribute(result.reason)
 
                 compileNonIncrementally(
-                    result.reason, allSourceFiles, args, projectDir, trackChangedFiles = changedFiles == null, messageCollector
+                    result.reason, allSourceFiles, args, fileLocations, trackChangedFiles = changedFiles == null, messageCollector
                 )
             }
             is ICResult.Failed -> {
@@ -139,7 +142,7 @@ abstract class IncrementalCompilerRunner<
                 reporter.addAttribute(result.reason)
 
                 compileNonIncrementally(
-                    result.reason, allSourceFiles, args, projectDir, trackChangedFiles = changedFiles == null, messageCollector
+                    result.reason, allSourceFiles, args, fileLocations, trackChangedFiles = changedFiles == null, messageCollector
                 )
             }
         }
@@ -173,8 +176,8 @@ abstract class IncrementalCompilerRunner<
         allSourceFiles: List<File>,
         changedFiles: ChangedFiles?,
         args: Args,
-        projectDir: File?,
-        messageCollector: MessageCollector
+        fileLocations: FileLocations?,
+        messageCollector: MessageCollector,
     ): ICResult {
         if (changedFiles is ChangedFiles.Unknown) {
             return ICResult.RequiresRebuild(UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
@@ -182,7 +185,7 @@ abstract class IncrementalCompilerRunner<
         changedFiles as ChangedFiles.Known?
 
         return createTransaction().runWithin(::incrementalCompilationExceptionTransformer) { transaction ->
-            val icContext = createIncrementalCompilationContext(projectDir, transaction)
+            val icContext = createIncrementalCompilationContext(fileLocations, transaction)
             val caches = createCacheManager(icContext, args).also {
                 // this way we make the transaction to be responsible for closing the caches manager
                 transaction.cachesManager = it
@@ -253,7 +256,7 @@ abstract class IncrementalCompilerRunner<
         rebuildReason: BuildAttribute,
         allSourceFiles: List<File>,
         args: Args,
-        projectDir: File?,
+        fileLocations: FileLocations?,
         trackChangedFiles: Boolean, // Whether we need to track changes to the source files or the build system already handles it
         messageCollector: MessageCollector,
     ): ExitCode {
@@ -266,7 +269,7 @@ abstract class IncrementalCompilerRunner<
             reporter.debug { "Cleaning ${outputDirsToClean.size} output directories" }
             cleanOrCreateDirectories(outputDirsToClean)
         }
-        val icContext = createIncrementalCompilationContext(projectDir, NonRecoverableCompilationTransaction())
+        val icContext = createIncrementalCompilationContext(fileLocations, NonRecoverableCompilationTransaction())
         return createCacheManager(icContext, args).use { caches ->
             if (trackChangedFiles) {
                 caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt

@@ -38,7 +38,7 @@ class InputsCache(
 
     fun removeOutputForSourceFiles(sources: Iterable<File>) {
         for (sourceFile in sources) {
-            sourceToOutputMap.remove(sourceFile).forEach {
+            sourceToOutputMap.getAndRemove(sourceFile)?.forEach {
                 icContext.reporter.debug { "Deleting $it on clearing cache for $sourceFile" }
                 icContext.transaction.deleteFile(it.toPath())
             }
@@ -46,7 +46,7 @@ class InputsCache(
     }
 
     fun getOutputForSourceFiles(sources: Iterable<File>): List<File> = sources.flatMap {
-        sourceToOutputMap[it]
+        sourceToOutputMap[it].orEmpty()
     }
 
     // generatedFiles can contain multiple entries with the same source file

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt

@@ -8,12 +8,14 @@ package org.jetbrains.kotlin.incremental.classpathDiff
 import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.build.report.DoNothingICReporter
 import org.jetbrains.kotlin.build.report.debug
-import org.jetbrains.kotlin.build.report.metrics.*
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
+import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
+import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.incremental.classpathDiff.BreadthFirstSearch.findReachableNodes
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrinkClasspath
 import org.jetbrains.kotlin.incremental.classpathDiff.ImpactedSymbolsComputer.computeImpactedSymbols
-import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter
 import org.jetbrains.kotlin.incremental.storage.ListExternalizer
 import org.jetbrains.kotlin.incremental.storage.loadFromFile
 import org.jetbrains.kotlin.name.ClassId
@@ -202,7 +204,7 @@ object ClasspathChangesComputer {
     ): ProgramSymbolSet {
         val workingDir =
             FileUtil.createTempDirectory(this::class.java.simpleName, "_WorkingDir_${UUID.randomUUID()}", /* deleteOnExit */ true)
-        val icContext = IncrementalCompilationContext(pathConverter = FileToAbsolutePathConverter)
+        val icContext = IncrementalCompilationContext()
         val incrementalJvmCache = IncrementalJvmCache(workingDir, icContext, null)
 
         // Step 1:

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt

@@ -14,15 +14,15 @@ class SourceToOutputFilesMap(
     icContext: IncrementalCompilationContext,
 ) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) {
     operator fun set(sourceFile: File, outputFiles: Collection<File>) {
-        storage[pathConverter.toPath(sourceFile)] = outputFiles.map(pathConverter::toPath)
+        storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] = outputFiles.map(icContext.pathConverterForOutputFiles::toPath)
     }
 
-    operator fun get(sourceFile: File): Collection<File> =
-        storage[pathConverter.toPath(sourceFile)].orEmpty().map(pathConverter::toFile)
+    operator fun get(sourceFile: File): Collection<File>? =
+        storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)]?.map(icContext.pathConverterForOutputFiles::toFile)
 
     override fun dumpValue(value: Collection<String>) =
         value.dumpCollection()
 
-    fun remove(file: File): Collection<File> =
-        get(file).also { storage.remove(pathConverter.toPath(file)) }
+    fun getAndRemove(file: File): Collection<File>? =
+        get(file).also { storage.remove(icContext.pathConverterForSourceFiles.toPath(file)) }
 }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/BuildDiffsStorageTest.kt

@@ -28,7 +28,7 @@ import java.util.*
 class BuildDiffsStorageTest {
     lateinit var storageFile: File
     private val random = Random(System.currentTimeMillis())
-    private val icContext = IncrementalCompilationContext(null)
+    private val icContext = IncrementalCompilationContext()
 
     @Before
     fun setUp() {

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt

@@ -7,8 +7,7 @@ package org.jetbrains.kotlin.incremental.snapshots
 
 import org.jetbrains.kotlin.TestWithWorkingDir
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
-import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
-import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
+import org.jetbrains.kotlin.incremental.storage.RelocatableFileToPathConverter
 import org.junit.After
 import org.junit.Assert.assertArrayEquals
 import org.junit.Before
@@ -24,9 +23,9 @@ class FileSnapshotMapTest : TestWithWorkingDir() {
         super.setUp()
         val caches = File(workingDir, "caches").apply { mkdirs() }
         val snapshotMapFile = File(caches, "snapshots.tab")
-        val pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile))
+        val pathConverter = RelocatableFileToPathConverter((workingDir.canonicalFile))
         val icContext = IncrementalCompilationContext(
-            pathConverter = pathConverter
+            pathConverterForSourceFiles = pathConverter
         )
         snapshotMap = FileSnapshotMap(snapshotMapFile, icContext)
     }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt

@@ -5,131 +5,105 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
-import org.jetbrains.kotlin.TestWithWorkingDir
+import junit.framework.TestCase.assertEquals
+import junit.framework.TestCase.assertNull
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.junit.After
-import org.junit.Assert.assertArrayEquals
 import org.junit.Before
+import org.junit.Rule
 import org.junit.Test
+import org.junit.rules.TemporaryFolder
 import java.io.File
-import kotlin.properties.Delegates
+import kotlin.test.assertFailsWith
 
-class SourceToOutputFilesMapTest : TestWithWorkingDir() {
-    private var stofMap: SourceToOutputFilesMap by Delegates.notNull()
-    private var pathConverter: FileToPathConverter by Delegates.notNull()
+class SourceToOutputFilesMapTest {
+
+    @get:Rule
+    val tmpDir = TemporaryFolder()
+
+    private lateinit var srcDir: File
+    private lateinit var classesDir: File
+
+    private lateinit var stofMap: SourceToOutputFilesMap
+
+    private lateinit var fooDotKt: File
+    private lateinit var fooDotClass: File
 
     @Before
-    override fun setUp() {
-        super.setUp()
-        val caches = File(workingDir, "caches").apply { mkdirs() }
-        val stofMapFile = File(caches, "stof.tab")
-        pathConverter = IncrementalFileToPathConverter((workingDir.canonicalFile))
-        val icContext = IncrementalCompilationContext(
-            pathConverter = pathConverter
+    fun setUp() {
+        val workingDir = tmpDir.root
+
+        srcDir = workingDir.resolve("src")
+        classesDir = workingDir.resolve("classes")
+
+        stofMap = SourceToOutputFilesMap(
+            storageFile = workingDir.resolve("stof.tab"),
+            icContext = IncrementalCompilationContext(
+                pathConverterForSourceFiles = RelocatableFileToPathConverter(srcDir),
+                pathConverterForOutputFiles = RelocatableFileToPathConverter(classesDir),
+            )
         )
-        stofMap = SourceToOutputFilesMap(stofMapFile, icContext)
+
+        fooDotKt = srcDir.resolve("Foo.kt")
+        fooDotClass = classesDir.resolve("Foo.class")
     }
 
     @After
-    override fun tearDown() {
+    fun tearDown() {
         stofMap.flush(false)
         stofMap.closeForTest()
-        super.tearDown()
     }
 
     @Test
-    fun testEmptyGetReturnsEmpty() {
-        assertTrue(stofMap.get(File("")).isEmpty())
+    fun testNoSetGetReturnsNull() {
+        assertNull(stofMap[fooDotKt])
     }
 
     @Test
-    fun testSetGetOneReturnsOne() {
-        stofMap.set(
-            File(""),
-            listOf(File("one").canonicalFile))
-        assertEquals(
-            listOf(File("one").canonicalFile),
-            stofMap.get(File("")))
+    fun testSetOneGetReturnsOne() {
+        stofMap[fooDotKt] = listOf(fooDotClass)
+
+        assertEquals(listOf(fooDotClass), stofMap[fooDotKt])
     }
 
     @Test
-    fun testSetDupeReturnsUnique() {
-        stofMap.set(
-            File(""),
-            listOf(File("one").canonicalFile, File("one").canonicalFile, File("one").canonicalFile))
+    fun testSetDupeGetReturnsUnique() {
+        stofMap[fooDotKt] = listOf(fooDotClass, fooDotClass)
 
-        assertEquals(
-            listOf(File("one").canonicalFile),
-            stofMap.get(File("")))
+        assertEquals(listOf(fooDotClass), stofMap[fooDotKt])
     }
 
     @Test
-    fun testSetOverwriteReturnsNew() {
-        stofMap.set(
-            File(""),
-            listOf(File("old").canonicalFile, File("old").canonicalFile, File("old").canonicalFile))
-        stofMap.set(
-            File(""),
-            listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile))
+    fun testSetOverwriteGetReturnsNew() {
+        val fooKtDotClass = classesDir.resolve("FooKt.class")
+        stofMap[fooDotKt] = listOf(fooDotClass)
+        stofMap[fooDotKt] = listOf(fooKtDotClass)
 
-        assertArrayEquals(
-            listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(),
-            stofMap.get(File("")).toSortedPaths())
+        assertEquals(listOf(fooKtDotClass), stofMap[fooDotKt])
     }
 
     @Test
-    fun testRelativeInReturnsAbsolute() {
-        stofMap.set(
-            File(""),
-            listOf(File("one"), File("two"), File("three")))
-
-        assertArrayEquals(
-            listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(),
-            stofMap.get(File("")).toSortedPaths()
-        )
-    }
-
-    @Test
-    fun testSetRelativeGetAbsolute() {
-        stofMap.set(
-            File("blah"),
-            listOf(File("one"), File("two"), File("three")))
-
-        assertArrayEquals(
-            listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(),
-            stofMap.get(File("blah").canonicalFile).toSortedPaths()
-        )
-    }
-
-    @Test
-    fun testSetRemove() {
-        stofMap.set(
-            File("blah"),
-            listOf(File("one"), File("two"), File("three")))
-
-        assertArrayEquals(
-            listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(),
-            stofMap.remove(File("blah")).toSortedPaths()
-        )
-        assertTrue(stofMap.get(File("blah")).isEmpty())
-    }
-
-    @Test
-    fun testSetRemoveLoop() {
-        repeat(5) {
-            stofMap.set(
-                File("blah"),
-                listOf(File("one"), File("two"), File("three"))
-            )
-
-            assertArrayEquals(
-                listOf(File("one").canonicalFile, File("two").canonicalFile, File("three").canonicalFile).toSortedPaths(),
-                stofMap.remove(File("blah")).toSortedPaths()
-            )
-            assertTrue(stofMap.get(File("blah")).isEmpty())
+    fun testSetRelativeFails() {
+        assertFailsWith<IllegalStateException> {
+            stofMap[fooDotKt] = listOf(File("relativePath"))
         }
     }
 
-    private fun Iterable<File>.toSortedPaths(): Array<String> =
-        map { it.canonicalPath }.sorted().toTypedArray()
+    @Test
+    fun testGetRelativeFails() {
+        stofMap[fooDotKt] = listOf(fooDotClass)
+
+        assertFailsWith<IllegalStateException> {
+            stofMap[fooDotKt.relativeTo(srcDir)]
+        }
+    }
+
+    @Test
+    fun testGetAndRemove() {
+        stofMap[fooDotKt] = listOf(fooDotClass)
+
+        assertEquals(listOf(fooDotClass), stofMap.getAndRemove(fooDotKt))
+        assertNull(stofMap[fooDotKt])
+    }
+
 }