[caches] Make output for cache a directory

(cherry picked from commit 9850bb322bee2bc98fc453a440e41af0dac0b0f2)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt

@@ -23,34 +23,39 @@ class CachedLibraries(
         enum class Kind { DYNAMIC, STATIC }
     }
 
+    private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? {
+        // See Linker.renameOutput why is it ok to have an empty cache directory.
+        if (cacheDir.listFilesOrEmpty.isEmpty()) return null
+        val baseName = getCachedLibraryName(library)
+        val dynamicFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE))
+        val staticFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE))
+
+        if (dynamicFile.exists && staticFile.exists)
+            error("Both dynamic and static caches files cannot be in the same directory." +
+                    " Library: ${library.libraryName}, path to cache: ${cacheDir.absolutePath}")
+        return when {
+            dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath)
+            staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath)
+            else -> error("No cache found for library ${library.libraryName} at ${cacheDir.absolutePath}")
+        }
+    }
+
     private val allCaches: Map<KotlinLibrary, Cache> = allLibraries.mapNotNull { library ->
         val explicitPath = explicitCaches[library]
 
         val cache = if (explicitPath != null) {
-            val kind = when {
+            selectCache(library, File(explicitPath))
-                explicitPath.endsWith(target.family.dynamicSuffix) -> Cache.Kind.DYNAMIC
+                    ?: error("No cache found for library ${library.libraryName} at $explicitPath")
-                explicitPath.endsWith(target.family.staticSuffix) -> Cache.Kind.STATIC
-                else -> error("unexpected cache: $explicitPath")
-            }
-            Cache(kind, explicitPath)
         } else {
             implicitCacheDirectories.firstNotNullResult { dir ->
-                val baseName = getCachedLibraryName(library)
+                selectCache(library, dir.child(getCachedLibraryName(library)))
-                val dynamicFile = dir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE))
-                val staticFile = dir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE))
-
-                when {
-                    dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath)
-                    staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath)
-                    else -> null
-                }
             }
         }
 
         cache?.let { library to it }
     }.toMap()
 
-    fun getArtifactName(baseName: String, kind: CompilerOutputKind) =
+    private fun getArtifactName(baseName: String, kind: CompilerOutputKind) =
             "${kind.prefix(target)}$baseName${kind.suffix(target)}"
 
     fun isLibraryCached(library: KotlinLibrary): Boolean =

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt

@@ -5,7 +5,6 @@ import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.LinkerOutputKind
-import org.jetbrains.kotlin.backend.konan.files.renameAtomic
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 
 internal fun determineLinkerOutput(context: Context): LinkerOutputKind =
@@ -45,21 +44,21 @@ internal class Linker(val context: Context) {
 
         val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten()
 
+        if (context.config.produce.isCache)
+            context.config.outputFiles.tempCacheDirectory!!.mkdirs()
         runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags)
+
         renameOutput()
     }
 
     private fun renameOutput() {
         if (context.config.produce.isCache) {
             val outputFiles = context.config.outputFiles
-            val outputFile = java.io.File(outputFiles.mainFileMangled)
+            // For caches the output file is a directory. It might be created by someone else,
-            val outputDsymBundle = java.io.File(outputFiles.mainFileMangled + ".dSYM")
+            // We have to delete it in order to the next renaming operation to succeed.
-            if (renameAtomic(outputFile.absolutePath, outputFiles.mainFile, /* replaceExisting = */ false))
+            java.io.File(outputFiles.mainFile).delete()
-                outputDsymBundle.renameTo(java.io.File(outputFiles.mainFile + ".dSYM"))
+            if (!java.io.File(outputFiles.tempCacheDirectory!!.absolutePath).renameTo(java.io.File(outputFiles.mainFile)))
-            else {
+                outputFiles.tempCacheDirectory.deleteRecursively()
-                outputFile.delete()
-                outputDsymBundle.deleteRecursively()
-            }
         }
     }
 
@@ -96,7 +95,7 @@ internal class Linker(val context: Context) {
             } else {
                 emptyList()
             }
-            executable = context.config.outputFiles.mainFileMangled
+            executable = context.config.outputFiles.nativeBinaryFile
         } else {
             val framework = File(context.config.outputFile)
             val dylibName = framework.name.removeSuffix(".framework")
@@ -127,7 +126,7 @@ internal class Linker(val context: Context) {
                             caches.dynamic +
                             libraryProvidedLinkerFlags + additionalLinkerArgs,
                     optimize = optimize, debug = debug, kind = linkerOutput,
-                    outputDsymBundle = context.config.outputFiles.mainFileMangled + ".dSYM",
+                    outputDsymBundle = context.config.outputFiles.symbolicInfoFile,
                     needsProfileLibrary = needsProfileLibrary).forEach {
                 it.logWith(context::log)
                 it.execute()

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt

@@ -27,33 +27,41 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile
     /**
      * Header file for dynamic library.
      */
-    val cAdapterHeader      by lazy { File("${outputName}_api.h") }
+    val cAdapterHeader by lazy { File("${outputName}_api.h") }
-    val cAdapterDef         by lazy { File("${outputName}.def") }
+    val cAdapterDef    by lazy { File("${outputName}.def") }
 
     /**
      * Main compiler's output file.
      */
-    val mainFile = outputName
+    val mainFile =
-            .prefixBaseNameIfNeeded(prefix)
+            if (produce.isCache)
-            .suffixIfNeeded(suffix)
+                outputName
+            else
+                outputName.fullOutputName()
 
-    val mainFileMangled = if (!produce.isCache) mainFile else {
+    val tempCacheDirectory =
-        (outputName + Random.nextLong().toString())
+            if (produce.isCache)
-                .prefixBaseNameIfNeeded(prefix)
+                File(outputName + Random.nextLong().toString())
-                .suffixIfNeeded(suffix)
+            else null
-    }
 
-    private fun String.prefixBaseNameIfNeeded(prefix: String): String {
+    val nativeBinaryFile =
-        return if (produce.isCache)
+            if (produce.isCache)
-            prefixBaseNameAlways(prefix)
+                tempCacheDirectory!!.child(File(outputName.fullOutputName()).absoluteFile.name).absolutePath
-        else prefixBaseNameIfNot(prefix)
+            else mainFile
-    }
 
-    private fun String.suffixIfNeeded(prefix: String): String {
+    val symbolicInfoFile = "$nativeBinaryFile.dSYM"
-        return if (produce.isCache)
+
-            suffixAlways(prefix)
+    private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix)
-        else suffixIfNot(prefix)
+
-    }
+    private fun String.prefixBaseNameIfNeeded(prefix: String) =
+            if (produce.isCache)
+                prefixBaseNameAlways(prefix)
+            else prefixBaseNameIfNot(prefix)
+
+    private fun String.suffixIfNeeded(prefix: String) =
+            if (produce.isCache)
+                suffixAlways(prefix)
+            else suffixIfNot(prefix)
 
     private fun String.prefixBaseNameAlways(prefix: String): String {
         val file = File(this).absoluteFile

build-tools/src/main/kotlin/org/jetbrains/kotlin/CacheTesting.kt

@@ -25,7 +25,7 @@ fun configureCacheTesting(project: Project): CacheTesting? {
     val target = project.testTarget
 
     val cacheDir = project.file("${project.buildDir}/cache")
-    val cacheFile = "$cacheDir/${cacheKind.prefix(target)}stdlib-cache${cacheKind.suffix(target)}"
+    val cacheFile = "$cacheDir/stdlib-cache"
     val dist = project.kotlinNativeDist
     val stdlib = "$dist/klib/common/stdlib"
     val compilerArgs = listOf("-Xcached-library=$stdlib,$cacheFile")

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt

@@ -31,15 +31,11 @@ open class KonanCacheTask: DefaultTask() {
     val cacheDirectory: File
         get() = cacheRoot.resolve("$target-g$cacheKind")
 
-    @get:OutputFile
+    @get:OutputDirectory
     protected val cacheFile: File
         get() {
-            val konanTarget = HostManager().targetByName(target)
             val klibName = originalKlib.nameWithoutExtension
-            val cachePrefix = cacheKind.outputKind.prefix(konanTarget)
+            return cacheDirectory.resolve("${klibName}-cache")
-            val cacheSuffix = cacheKind.outputKind.suffix(konanTarget)
-            val cacheName = "${cachePrefix}${klibName}-cache${cacheSuffix}"
-            return cacheDirectory.resolve(cacheName)
         }
 
     @Input

utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt

@@ -260,7 +260,7 @@ private fun generateLibrary(
     }
 }
 
-private fun getCacheFile(
+private fun getLibraryCacheDir(
         libraryName: String,
         target: KonanTarget,
         cacheDirectory: File,
@@ -279,14 +279,14 @@ private fun buildCache(
         rebuild: Boolean,
         logger: Logger
 ) = with(cacheInfo) {
-    val cacheFile = getCacheFile(def.name, target, cacheDirectory, cacheKind)
+    val libraryCacheDir = getLibraryCacheDir(def.name, target, cacheDirectory, cacheKind)
-    if (cacheFile.exists && !rebuild) {
+    if (libraryCacheDir.listFilesOrEmpty.isNotEmpty() && !rebuild) {
         logger.verbose("Skip precompiling ${def.name} as it's already precompiled")
         return
     }
 
     if (rebuild) {
-        cacheFile.delete()
+        libraryCacheDir.deleteRecursively()
     }
 
     val compilerArgs = arrayOf(
@@ -307,7 +307,7 @@ private fun buildStdlibCache(
         cacheInfo: CacheInfo,
         logger: Logger
 ) = with(cacheInfo) {
-    val stdlibCacheFile = getCacheFile("stdlib", target, cacheDirectory, cacheKind)
+    val stdlibCacheFile = getLibraryCacheDir("stdlib", target, cacheDirectory, cacheKind)
     if (stdlibCacheFile.exists) {
         logger.verbose("Skip precompiling standard library as it's already precompiled")
         return