Added property for overriding konan distribution location

#KT-50463 Fixed

Merge-request: KT-MR-10310
Merged-by: Dmitrii Krasnov <Dmitrii.Krasnov@jetbrains.com>

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt

@@ -44,9 +44,9 @@ private fun makeDependencyAssigner(targets: List<String>, defFiles: List<File>,
         CompositeDependencyAssigner(targets.map { makeDependencyAssignerForTarget(it, defFiles, runFromDaemon) })
 
 private fun makeDependencyAssignerForTarget(target: String, defFiles: List<File>, runFromDaemon: Boolean): SingleTargetDependencyAssigner {
-    val tool = prepareTool(target, KotlinPlatform.NATIVE, runFromDaemon)
     val cinteropArguments = CInteropArguments()
     cinteropArguments.argParser.parse(arrayOf())
+    val tool = prepareTool(target, KotlinPlatform.NATIVE, runFromDaemon, konanDataDir = cinteropArguments.konanDataDir)
     val libraries = defFiles.parallelStream().map {
         it to buildNativeLibrary(
                 tool,

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/CommandLine.kt

@@ -16,10 +16,7 @@
 
 package org.jetbrains.kotlin.native.interop.tool
 
-import kotlinx.cli.ArgParser
-import kotlinx.cli.ArgType
 import kotlinx.cli.*
-import org.jetbrains.kotlin.native.interop.gen.jvm.GenerationMode
 
 const val HEADER_FILTER_ADDITIONAL_SEARCH_PREFIX = "headerFilterAdditionalSearchPrefix"
 const val NODEFAULTLIBS_DEPRECATED = "nodefaultlibs"
@@ -35,6 +32,7 @@ const val FOREIGN_EXCEPTION_MODE = "Xforeign-exception-mode"
 const val DUMP_BRIDGES = "Xdump-bridges"
 const val DISABLE_EXCEPTION_PRETTIFIER = "Xdisable-exception-prettifier"
 const val USER_SETUP_HINT = "Xuser-setup-hint"
+const val KONAN_DATA_DIR = "Xkonan-data-dir"
 
 // TODO: unify camel and snake cases.
 // Possible solution is to accept both cases
@@ -74,6 +72,8 @@ open class CommonInteropArguments(val argParser: ArgParser) {
             fullName = "Xoverride-konan-properties",
             description = "Override konan.properties.values"
         ).multiple().delimiter(";")
+    val konanDataDir by argParser.option(ArgType.String, KONAN_DATA_DIR,
+            description = "Path to konan and dependencies root folder")
 }
 
 open class CInteropArguments(argParser: ArgParser =

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt

@@ -22,8 +22,8 @@ import org.jetbrains.kotlin.konan.target.AbstractToolConfig
 import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform
 import org.jetbrains.kotlin.native.interop.indexer.Language
 
-class ToolConfig(userProvidedTargetName: String?, flavor: KotlinPlatform, propertyOverrides: Map<String, String>)
-    : AbstractToolConfig(KonanHomeProvider.determineKonanHome(), userProvidedTargetName, propertyOverrides) {
+class ToolConfig(userProvidedTargetName: String?, flavor: KotlinPlatform, propertyOverrides: Map<String, String>, konanDataDir: String? = null)
+    : AbstractToolConfig(KonanHomeProvider.determineKonanHome(), userProvidedTargetName, propertyOverrides, konanDataDir) {
 
     val clang = when (flavor) {
         KotlinPlatform.JVM -> platform.clangForJni

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -259,7 +259,7 @@ private fun processCLib(
         cinteropArguments.argParser.printError("-def or -pkg should be provided!")
     }
 
-    val tool = prepareTool(cinteropArguments.target, flavor, runFromDaemon, parseKeyValuePairs(cinteropArguments.overrideKonanProperties))
+    val tool = prepareTool(cinteropArguments.target, flavor, runFromDaemon, parseKeyValuePairs(cinteropArguments.overrideKonanProperties), konanDataDir = cinteropArguments.konanDataDir)
 
     val def = DefFile(defFile, tool.substitutions)
     val isLinkerOptsSetByUser = (cinteropArguments.linkerOpts.valueOrigin == ArgParser.ValueOrigin.SET_BY_USER) ||
@@ -480,7 +480,7 @@ private fun getLibraryResolver(
             repos,
             libraries.filter { it.contains(org.jetbrains.kotlin.konan.file.File.separator) },
             target,
-            Distribution(KonanHomeProvider.determineKonanHome())
+            Distribution(KonanHomeProvider.determineKonanHome(), konanDataDir = cinteropArguments.konanDataDir)
     ).libraryResolver()
 }
 
@@ -498,8 +498,8 @@ private fun resolveDependencies(
     ).getFullList(TopologicalLibraryOrder)
 }
 
-internal fun prepareTool(target: String?, flavor: KotlinPlatform, runFromDaemon: Boolean, propertyOverrides: Map<String, String> = emptyMap()) =
-        ToolConfig(target, flavor, propertyOverrides).also {
+internal fun prepareTool(target: String?, flavor: KotlinPlatform, runFromDaemon: Boolean, propertyOverrides: Map<String, String> = emptyMap(), konanDataDir: String? = null) =
+        ToolConfig(target, flavor, propertyOverrides, konanDataDir).also {
             if (!runFromDaemon) it.prepare() // Daemon prepares the tool himself. (See KonanToolRunner.kt)
         }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -42,7 +42,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
                 configuration.get(KonanConfigKeys.KONAN_HOME) ?: KonanHomeProvider.determineKonanHome(),
                 false,
                 configuration.get(KonanConfigKeys.RUNTIME_FILE),
-                overridenProperties
+                overridenProperties,
+                configuration.get(KonanConfigKeys.KONAN_DATA_DIR)
         )
     }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt

@@ -165,6 +165,7 @@ class KonanConfigKeys {
         val SERIALIZED_DEPENDENCIES: CompilerConfigurationKey<String?> = CompilerConfigurationKey.create("path to serialized dependencies for native linking")
         val SAVE_DEPENDENCIES_PATH: CompilerConfigurationKey<String?> = CompilerConfigurationKey.create("path to save serialized dependencies to")
         val SAVE_LLVM_IR_DIRECTORY: CompilerConfigurationKey<String?> = CompilerConfigurationKey.create("directory to store LLVM IR from phases")
+        val KONAN_DATA_DIR: CompilerConfigurationKey<String?> = CompilerConfigurationKey.create("directory for storing konan dependencies, cache and prebuilds")
     }
 }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt

@@ -306,6 +306,7 @@ fun CompilerConfiguration.setupFromArguments(arguments: K2NativeCompilerArgument
     putIfNotNull(SERIALIZED_DEPENDENCIES, parseSerializedDependencies(arguments, this@setupFromArguments))
     putIfNotNull(SAVE_DEPENDENCIES_PATH, arguments.saveDependenciesPath)
     putIfNotNull(SAVE_LLVM_IR_DIRECTORY, arguments.saveLlvmIrDirectory)
+    putIfNotNull(KONAN_DATA_DIR, arguments.konanDataDir)
 }
 
 private fun String.absoluteNormalizedFile() = java.io.File(this).absoluteFile.normalize()

kotlin-native/build.gradle

@@ -54,8 +54,13 @@ apply plugin: "kotlin.native.build-tools-conventions"
 
 ext {
     distDir = UtilsKt.getKotlinNativeDist(project)
-    dependenciesDir = DependencyProcessor.defaultDependenciesRoot
+    // TODO(Dmitrii Krasnov): remove else branch after KT-50463 will be in current bootstrap
+    def konanDataDir = project.findProperty('konan.data.dir')
+    dependenciesDir = konanDataDir != null ?
+            DependencyDirectories.INSTANCE.getDependenciesRoot(konanDataDir.toString())
+            : DependencyDirectories.defaultDependenciesRoot
     experimentalEnabled = project.hasProperty("org.jetbrains.kotlin.native.experimentalTargets")
+    // TODO(Dmitrii Krasnov): add konanDataDir to the buildDistribution after KT-50463 will be in current bootstrap
     platformManager = new PlatformManager(DistributionKt.buildDistribution(projectDir.absolutePath),
             ext.experimentalEnabled)
 

kotlin-native/dependencies/build.gradle

@@ -5,14 +5,10 @@
 import groovy.transform.stc.ClosureParams
 import groovy.transform.stc.FromString
 import org.jetbrains.kotlin.konan.target.*
-import static org.jetbrains.kotlin.konan.target.KonanTarget.*
-import org.jetbrains.kotlin.konan.util.Named
 import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
 import org.jetbrains.kotlin.konan.util.ArchiveType
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
 
-import static org.jetbrains.kotlin.konan.util.VisibleNamedKt.getVisibleName
-
 buildscript {
     apply from: "$rootDir/kotlin-native/gradle/kotlinGradlePlugin.gradle"
 

kotlin-native/klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolverByName
 import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.PlatformManager
+import org.jetbrains.kotlin.konan.util.DependencyDirectories
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import org.jetbrains.kotlin.konan.util.KonanHomeProvider
 import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION_WITH_DOT
@@ -93,7 +94,8 @@ object KlibToolLogger : Logger {
     override fun log(message: String) = println(message)
 }
 
-val defaultRepository = File(DependencyProcessor.localKonanDir.resolve("klib").absolutePath)
+// TODO(Dmitrii Krasnov): I'm not sure that we should put konan distribution dir here
+val defaultRepository = File(DependencyDirectories.localKonanDir.resolve("klib").absolutePath)
 
 open class ModuleDeserializer(val library: ByteArray) {
     protected val moduleHeader: KlibMetadataProtoBuf.Header

kotlin-native/konan/konan.properties

@@ -30,7 +30,6 @@ minimalXcodeVersion=12.5
 
 downloadingAttempts = 10
 downloadingAttemptIntervalMs = 3000
-homeDependencyCache = .konan/cache
 
 # Appendix that is used for smaller version of LLVM distributions.
 reducedLlvmAppendix = compact

kotlin-native/performance/build.gradle

@@ -43,6 +43,7 @@ project.logger.info("kotlin_root: $kotlin_root")
 globalProperties.load(new java.io.FileReader(project.file("$kotlin_root/gradle.properties")))
 ext.kotlinNativeVersionInResources = true
 
+// TODO(Dmitrii Krasnov): add konanDataDir to the buildDistribution after KT-50463 will be in current bootstrap
 def platformManager = new PlatformManager(DistributionKt.buildDistribution(UtilsKt.getKotlinNativeDist(project).path), false)
 def kotlinDist = null
 if (hasProperty("kotlin_dist")) {

kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanPlugin.kt

@@ -46,8 +46,8 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.buildDistribution
 import org.jetbrains.kotlin.konan.target.customerDistribution
-import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import org.jetbrains.kotlin.*
+import org.jetbrains.kotlin.konan.util.DependencyDirectories
 import java.io.File
 import javax.inject.Inject
 
@@ -166,7 +166,7 @@ internal fun Project.konanCompilerName(): String =
         "kotlin-native-${project.simpleOsName}-${project.konanVersion}"
 
 internal fun Project.konanCompilerDownloadDir(): String =
-        DependencyProcessor.localKonanDir.resolve(project.konanCompilerName()).absolutePath
+        DependencyDirectories.localKonanDir.resolve(project.konanCompilerName()).absolutePath
 
 // region Useful extensions and functions ---------------------------------------
 

kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt

@@ -119,13 +119,18 @@ fun generatePlatformLibraries(args: Array<String>) = usingNativeMemoryAllocator
             description = "Override konan.properties.values"
     ).multiple().delimiter(";")
 
+    val konanDataDir by argParser.option(ArgType.String,
+            fullName = "Xkonan-data-dir",
+            description = "Path to konan and dependencies root folder")
+
     argParser.parse(args)
 
     val distribution = Distribution(
             KonanHomeProvider.determineKonanHome(),
             onlyDefaultProfiles = false,
             runtimeFileOverride = null,
-            propertyOverrides = parseKeyValuePairs(overrideKonanProperties)
+            propertyOverrides = parseKeyValuePairs(overrideKonanProperties),
+            konanDataDir = konanDataDir
     )
 
     val platformManager = PlatformManager(distribution)
@@ -163,6 +168,11 @@ fun generatePlatformLibraries(args: Array<String>) = usingNativeMemoryAllocator
                     add("-Xoverride-konan-properties")
                     add(overrideKonanProperties.joinToString(";"))
                 }
+
+                konanDataDir?.let {
+                    add("-Xkonan-data-dir")
+                    add(it)
+                }
             }
     )
 

kotlin-native/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt

@@ -40,7 +40,9 @@ fun invokeInterop(flavor: String, args: Array<String>, runFromDaemon: Boolean):
     val repos = arguments.repo
     val targetRequest = if (arguments is CInteropArguments) arguments.target
         else (arguments as JSInteropArguments).target.toString()
-    val target = PlatformManager(KonanHomeProvider.determineKonanHome()).targetManager(targetRequest).target
+    val target = PlatformManager(
+            KonanHomeProvider.determineKonanHome(),
+            konanDataDir = arguments.konanDataDir).targetManager(targetRequest).target
 
     val cinteropArgsToCompiler = Interop().interop(flavor, args,
             InternalInteropOptions(generatedDir.absolutePath,