Rework compiler dependency management

And support for searching them locally and
downloading from internal server if KONAN_USE_INTERNAL_SERVER is set

Also:
Allow sysroot to change version minorly.

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/HeaderToIdMapper.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.native.interop.indexer
+
+import java.nio.file.Paths
+
+class HeaderToIdMapper(sysRoot: String) {
+    private val headerPathToId = mutableMapOf<String, HeaderId>()
+    private val sysRoot = Paths.get(sysRoot).normalize()
+
+    internal fun getHeaderId(filePath: String) = headerPathToId.getOrPut(filePath) {
+        val path = Paths.get(filePath)
+        val headerIdValue = if (path.startsWith(sysRoot)) {
+            val relative = sysRoot.relativize(path)
+            relative.toString()
+        } else {
+            headerContentsHash(filePath)
+        }
+        HeaderId(headerIdValue)
+    }
+}

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt

@@ -111,18 +111,13 @@ internal class NativeIndexImpl(val library: NativeLibrary) : NativeIndex() {
 
     }
 
-    private val headerPathToId = mutableMapOf<String, HeaderId>()
-
     internal fun getHeaderId(file: CXFile?): HeaderId {
         if (file == null) {
             return HeaderId("builtins")
         }
 
         val filePath = clang_getFileName(file).convertAndDispose()
-        return headerPathToId.getOrPut(filePath) {
+        return library.headerToIdMapper.getHeaderId(filePath)
-            val headerIdValue = headerContentsHash(filePath)
-            HeaderId(headerIdValue)
-        }
     }
 
     private fun getContainingFile(cursor: CValue<CXCursor>): CXFile? {

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -44,6 +44,7 @@ interface HeaderInclusionPolicy {
 data class NativeLibrary(val includes: List<String>,
                          val additionalPreambleLines: List<String>,
                          val compilerArgs: List<String>,
+                         val headerToIdMapper: HeaderToIdMapper,
                          val language: Language,
                          val excludeSystemLibs: Boolean, // TODO: drop?
                          val excludeDepdendentModules: Boolean,

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt

@@ -28,7 +28,7 @@ fun KonanProperties.defaultCompilerOpts(): List<String> {
     // TODO: eliminate this. below
     val targetToolchain = absoluteTargetToolchain
     val targetSysRoot = absoluteTargetSysRoot
-    val llvmHome = absolute(hostString("llvmHome"))
+    val llvmHome = absoluteLlvmHome
     val llvmVersion = hostString("llvmVersion")!!
 
     // StubGenerator passes the arguments to libclang which

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt

@@ -40,10 +40,11 @@ class ToolConfig(userProvidedTargetName: String?, userProvidedKonanProperties: S
         "target" to target.detailedName,
         "arch" to target.architecture.userName)
 
-    fun downloadDependencies() = maybeExecuteHelper(dependencies.absolutePath, 
+    fun downloadDependencies() = targetProperties.downloadDependencies()
-            properties, targetProperties.dependencies)
 
-    val llvmHome = targetProperties.absolute(targetProperties.hostString("llvmHome"))
+    val llvmHome = targetProperties.absoluteLlvmHome
+
+    val sysRoot get() = targetProperties.absoluteTargetSysRoot
 
     val defaultCompilerOpts = 
         targetProperties.defaultCompilerOpts()

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -349,6 +349,7 @@ private fun processLib(args: Map<String, List<String>>,
             includes = headerFiles,
             additionalPreambleLines = def.defHeaderLines,
             compilerArgs = compilerOpts,
+            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
             language = language,
             excludeSystemLibs = excludeSystemLibs,
             excludeDepdendentModules = excludeDependentModules,

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt

@@ -52,15 +52,8 @@ class Distribution(val targetManager: TargetManager,
     val dependenciesDir = DependencyProcessor.defaultDependenciesRoot.absolutePath
 
     val targetProperties = KonanProperties(target, properties, dependenciesDir)
-    val hostProperties = if (target == host) {
-        targetProperties 
-    } else {
-        KonanProperties(host, properties, dependenciesDir)
-    }
-    val dependencies = targetProperties.dependencies
 
-    val llvmHome = hostProperties.absoluteLlvmHome
+    val llvmHome = targetProperties.absoluteLlvmHome
-    val hostSysRoot = hostProperties.absoluteTargetSysRoot
     val llvmBin = "$llvmHome/bin"
     val llvmLib = "$llvmHome/lib"
     val llvmLto = "$llvmBin/llvm-lto"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -54,7 +54,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
     private fun Distribution.prepareDependencies(checkDependencies: Boolean) {
         if (checkDependencies) {
-            DependencyProcessor(java.io.File(dependenciesDir), targetProperties).run()
+            targetProperties.downloadDependencies()
         }
     }
 

backend.native/konan.properties

@@ -17,6 +17,9 @@
 # TODO: Do we need a $variable substitution mechanism here?
 dependenciesUrl = https://download.jetbrains.com/kotlin/native
 
+# In order of preference:
+dependencyProfiles = default alt
+
 # Don't check dependencies on a server.
 # If true, dependency downloader will throw an exception if a dependency isn't found in konan.home.
 airplaneMode = false

build.gradle

@@ -26,6 +26,16 @@ ext {
     konanPropertiesFile = project(':backend.native').file('konan.properties')
     konanProperties = PropertiesKt.loadProperties(konanPropertiesFile.absolutePath)
 
+    String DEPENDENCY_PROFILES_KEY = "dependencyProfiles"
+    def dependencyProfiles = konanProperties.getProperty(DEPENDENCY_PROFILES_KEY)
+    if (dependencyProfiles != "default alt")
+        throw new Error("unexpected $DEPENDENCY_PROFILES_KEY value: expected 'default alt', got '$dependencyProfiles'")
+
+    // Force build to use only 'default' profile:
+    konanProperties.setProperty(DEPENDENCY_PROFILES_KEY, "default")
+    // TODO: it actually affects only resolution made in :dependencies,
+    // that's why we assume that 'default' profile comes first (and check this above).
+
     distDir = file('dist')
     dependenciesDir = DependencyProcessor.defaultDependenciesRoot
     clangManager = new ClangManager(konanProperties, dependenciesDir.absolutePath)

dependencies/build.gradle

@@ -139,7 +139,7 @@ task update_kotlin_compiler(type: DefaultTask) {
 
 abstract class NativeDep extends DefaultTask {
     protected final String hostSystem = TargetManager.longerSystemName();
-    String baseUrl = "https://jetbrains.bintray.com/kotlin-native-dependencies"
+    static final String baseUrl = "https://jetbrains.bintray.com/kotlin-native-dependencies"
 
     @Input
     abstract String getFileName()
@@ -254,12 +254,19 @@ TargetManager.enabled.each { target ->
         }
     }
 
+    // Also resolves all dependencies:
+    final DependencyProcessor dependencyProcessor = new DependencyProcessor(
+            project.rootProject.ext.dependenciesDir,
+            rootProject.ext.konanProperties,
+            konanProperties.dependencies,
+            HelperNativeDep.baseUrl,
+            false
+    )
+
     DependencyKind.values().each { kind ->
         def dir = kind.getDirectory(konanProperties)
         if (dir != null) {
-            def path = rootProject.ext.dependenciesDir.toPath()
+            String path = dependencyProcessor.resolveRelative(dir).canonicalPath
-                    .resolve(dir).toFile()
-                    .canonicalPath
             rootProject.ext.set(kind.getPropertyName(target), path)
         }
     }

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanProperties.kt

@@ -18,9 +18,14 @@ package org.jetbrains.kotlin.konan.properties
 
 import org.jetbrains.kotlin.konan.file.*
 import org.jetbrains.kotlin.konan.target.*
+import org.jetbrains.kotlin.konan.util.DependencyProcessor
 
 class KonanProperties(val target: KonanTarget, val properties: Properties, val baseDir: String? = null) {
 
+    fun downloadDependencies() {
+        dependencyProcessor!!.run()
+    }
+
     fun targetString(key: String): String? 
         = properties.targetString(key, target)
     fun targetList(key: String): List<String> 
@@ -34,6 +39,8 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b
     fun hostTargetList(key: String): List<String> 
         = properties.hostTargetList(key, target)
 
+    val llvmHome get() = hostString("llvmHome")
+
     // TODO: Delegate to a map?
     val llvmLtoNooptFlags get() = targetList("llvmLtoNooptFlags")
     val llvmLtoOptFlags get() = targetList("llvmLtoOptFlags")
@@ -51,12 +58,12 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b
     val libffiDir get() = targetString("libffiDir")
     val gccToolchain get() = targetString("gccToolchain")
     val targetArg get() = targetString("quadruple")
-    val llvmHome get() = targetString("llvmHome")
     // Notice: these ones are host-target.
     val targetToolchain get() = hostTargetString("targetToolchain")
     val dependencies get() = hostTargetList("dependencies")
 
-    fun absolute(value: String?) = "${baseDir!!}/${value!!}"
+    private fun absolute(value: String?): String =
+            dependencyProcessor!!.resolveRelative(value!!).absolutePath
 
     val absoluteTargetSysRoot get() = absolute(targetSysRoot)
     val absoluteTargetToolchain get() = absolute(targetToolchain)
@@ -75,4 +82,7 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b
         }
 
     val osVersionMin: String? get() = targetString("osVersionMin")
+
+    private val dependencyProcessor = baseDir?.let { DependencyProcessor(java.io.File(it), this) }
+
 }

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt

@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.konan.util
 import org.jetbrains.kotlin.konan.file.use
 import org.jetbrains.kotlin.konan.properties.KonanProperties
 import org.jetbrains.kotlin.konan.properties.Properties
+import org.jetbrains.kotlin.konan.properties.propertyList
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.RandomAccessFile
@@ -45,6 +46,26 @@ private val Properties.downloadingAttemptIntervalMs : Long
 private val Properties.homeDependencyCache : String
     get() = getProperty("homeDependencyCache") ?: DependencyProcessor.DEFAULT_HOME_DEPENDENCY_CACHE
 
+private fun Properties.findCandidates(dependencies: List<String>): Map<String, List<DependencySource>> {
+    val dependencyProfiles = this.propertyList("dependencyProfiles")
+    return dependencies.map { dependency ->
+        dependency to dependencyProfiles.flatMap { profile ->
+            val candidateSpecs = propertyList("$dependency.$profile")
+            if (profile == "default" && candidateSpecs.isEmpty()) {
+                listOf(DependencySource.Remote.Public)
+            } else {
+                candidateSpecs.map { candidateSpec ->
+                    when (candidateSpec) {
+                        "remote:public" -> DependencySource.Remote.Public
+                        "remote:internal" -> DependencySource.Remote.Internal
+                        else -> DependencySource.Local(File(candidateSpec))
+                    }
+                }
+            }
+        }
+    }.toMap()
+}
+
 
 private val KonanProperties.dependenciesUrl : String            get() = properties.dependenciesUrl
 private val KonanProperties.airplaneMode : Boolean              get() = properties.airplaneMode
@@ -52,6 +73,17 @@ private val KonanProperties.downloadingAttempts : Int           get() = properti
 private val KonanProperties.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs
 private val KonanProperties.homeDependencyCache : String        get() = properties.homeDependencyCache
 
+private val internalDependenciesUrl = "http://repo.labs.intellij.net/kotlin-native"
+
+sealed class DependencySource {
+    data class Local(val path: File) : DependencySource()
+
+    sealed class Remote : DependencySource() {
+        object Public : Remote()
+        object Internal : Remote()
+    }
+}
+
 // TODO: Try to use some dependency management system (Ivy?)
 // TODO: Maybe rename.
 /**
@@ -60,13 +92,13 @@ private val KonanProperties.homeDependencyCache : String        get() = properti
  */
 class DependencyProcessor(dependenciesRoot: File,
                           val dependenciesUrl: String,
-                          val dependencies: Collection<String>,
+                          dependencyToCandidates: Map<String, List<DependencySource>>,
                           homeDependencyCache: String = DEFAULT_HOME_DEPENDENCY_CACHE,
                           val airplaneMode: Boolean = false,
                           maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
                           attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS,
                           customProgressCallback: ProgressCallback? = null,
-                          val keepUnstable:Boolean = true) {
+                          val keepUnstable: Boolean = true) {
 
     val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
     val cacheDirectory = System.getProperty("user.home")?.let {
@@ -101,7 +133,7 @@ class DependencyProcessor(dependenciesRoot: File,
                 keepUnstable:Boolean = true) : this(
             dependenciesRoot,
             dependenciesUrl,
-            dependencies,
+            dependencyToCandidates = properties.findCandidates(dependencies),
             airplaneMode = properties.airplaneMode,
             maxAttempts = properties.downloadingAttempts,
             attemptIntervalMs = properties.downloadingAttemptIntervalMs,
@@ -137,13 +169,13 @@ class DependencyProcessor(dependenciesRoot: File,
         }
     }
 
-    private fun processDependency(dependency: String) {
+    private fun downloadDependency(dependency: String, baseUrl: String) {
         val depDir = File(dependenciesDirectory, dependency)
         val depName = depDir.name
 
         val fileName = "$depName.$archiveExtension"
         val archive = cacheDirectory.resolve(fileName)
-        val url = URL("$dependenciesUrl/$fileName")
+        val url = URL("$baseUrl/$fileName")
 
         val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
         if (extractedDependencies.contains(depName) &&
@@ -190,10 +222,57 @@ class DependencyProcessor(dependenciesRoot: File,
             get() = Paths.get(System.getProperty("user.home")).resolve(".konan/dependencies").toFile()
     }
 
+    private fun internalServerIsAvailable(): Boolean = System.getenv("KONAN_USE_INTERNAL_SERVER") != null
+
+    private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) ->
+        val candidate = candidates.asSequence().mapNotNull { candidate ->
+            when (candidate) {
+                is DependencySource.Local -> candidate.takeIf { it.path.exists() }
+                DependencySource.Remote.Public -> candidate
+                DependencySource.Remote.Internal -> candidate.takeIf { internalServerIsAvailable() }
+            }
+        }.firstOrNull()
+
+        candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}")
+
+        dependency to candidate
+    }.toMap()
+
+    private fun resolveDependency(dependency: String): File {
+        val candidate = resolvedDependencies[dependency]
+        return when (candidate) {
+            is DependencySource.Local -> candidate.path
+            is DependencySource.Remote -> File(dependenciesDirectory, dependency)
+            null -> error("$dependency not declared as dependency")
+        }
+    }
+
+    fun resolveRelative(relative: String): File {
+        val path = Paths.get(relative)
+        if (path.isAbsolute) error("not a relative path: $relative")
+
+        val dependency = path.first().toString()
+        return resolveDependency(dependency).let {
+            if (path.nameCount > 1) {
+                it.toPath().resolve(path.subpath(1, path.nameCount)).toFile()
+            } else {
+                it
+            }
+        }
+    }
+
     fun run() = lock.withLock {
         RandomAccessFile(lockFile, "rw").channel.lock().use {
-            dependencies.forEach {
+            resolvedDependencies.forEach { (dependency, candidate) ->
-                processDependency(it)
+                val baseUrl = when (candidate) {
+                    is DependencySource.Local -> null
+                    DependencySource.Remote.Public -> dependenciesUrl
+                    DependencySource.Remote.Internal -> internalDependenciesUrl
+                }
+                // TODO: consider using different caches for different remotes.
+                if (baseUrl != null) {
+                    downloadDependency(dependency, baseUrl)
+                }
             }
         }
     }

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/tasks/KonanCompilerDownloadTask.kt

@@ -22,6 +22,7 @@ import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
+import org.jetbrains.kotlin.konan.util.DependencySource
 import java.io.File
 import java.io.IOException
 
@@ -60,7 +61,11 @@ open class KonanCompilerDownloadTask : DefaultTask() {
                 }
                 val konanCompiler = project.konanCompilerName()
                 logger.info("Downloading Kotlin/Native compiler from $downloadUrlDirectory/$konanCompiler into $KONAN_PARENT_DIR")
-                DependencyProcessor(File(KONAN_PARENT_DIR), downloadUrlDirectory, listOf(konanCompiler)).run()
+                DependencyProcessor(
+                        File(KONAN_PARENT_DIR),
+                        downloadUrlDirectory,
+                        mapOf(konanCompiler to listOf(DependencySource.Remote.Public))
+                ).run()
             } catch (e: IOException) {
                 throw GradleScriptException("Cannot download Kotlin/Native compiler", e)
             }