diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/HeaderToIdMapper.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/HeaderToIdMapper.kt new file mode 100644 index 00000000000..ded80276074 --- /dev/null +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/HeaderToIdMapper.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.native.interop.indexer + +import java.nio.file.Paths + +class HeaderToIdMapper(sysRoot: String) { + private val headerPathToId = mutableMapOf() + private val sysRoot = Paths.get(sysRoot).normalize() + + internal fun getHeaderId(filePath: String) = headerPathToId.getOrPut(filePath) { + val path = Paths.get(filePath) + val headerIdValue = if (path.startsWith(sysRoot)) { + val relative = sysRoot.relativize(path) + relative.toString() + } else { + headerContentsHash(filePath) + } + HeaderId(headerIdValue) + } +} diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt index fa8ab2513c5..191e093f388 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt @@ -111,18 +111,13 @@ internal class NativeIndexImpl(val library: NativeLibrary) : NativeIndex() { } - private val headerPathToId = mutableMapOf() - internal fun getHeaderId(file: CXFile?): HeaderId { if (file == null) { return HeaderId("builtins") } val filePath = clang_getFileName(file).convertAndDispose() - return headerPathToId.getOrPut(filePath) { - val headerIdValue = headerContentsHash(filePath) - HeaderId(headerIdValue) - } + return library.headerToIdMapper.getHeaderId(filePath) } private fun getContainingFile(cursor: CValue): CXFile? { diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt index de6bf4a2f4b..e2cb855d544 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt @@ -44,6 +44,7 @@ interface HeaderInclusionPolicy { data class NativeLibrary(val includes: List, val additionalPreambleLines: List, val compilerArgs: List, + val headerToIdMapper: HeaderToIdMapper, val language: Language, val excludeSystemLibs: Boolean, // TODO: drop? val excludeDepdendentModules: Boolean, diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt index 2ef41519857..030849125f2 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ClangDefaults.kt @@ -28,7 +28,7 @@ fun KonanProperties.defaultCompilerOpts(): List { // TODO: eliminate this. below val targetToolchain = absoluteTargetToolchain val targetSysRoot = absoluteTargetSysRoot - val llvmHome = absolute(hostString("llvmHome")) + val llvmHome = absoluteLlvmHome val llvmVersion = hostString("llvmVersion")!! // StubGenerator passes the arguments to libclang which diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt index df53df4d828..d5eb628db58 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/ToolConfig.kt @@ -40,10 +40,11 @@ class ToolConfig(userProvidedTargetName: String?, userProvidedKonanProperties: S "target" to target.detailedName, "arch" to target.architecture.userName) - fun downloadDependencies() = maybeExecuteHelper(dependencies.absolutePath, - properties, targetProperties.dependencies) + fun downloadDependencies() = targetProperties.downloadDependencies() - val llvmHome = targetProperties.absolute(targetProperties.hostString("llvmHome")) + val llvmHome = targetProperties.absoluteLlvmHome + + val sysRoot get() = targetProperties.absoluteTargetSysRoot val defaultCompilerOpts = targetProperties.defaultCompilerOpts() diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index de10bb07ee3..6263d06a082 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -349,6 +349,7 @@ private fun processLib(args: Map>, includes = headerFiles, additionalPreambleLines = def.defHeaderLines, compilerArgs = compilerOpts, + headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot), language = language, excludeSystemLibs = excludeSystemLibs, excludeDepdendentModules = excludeDependentModules, diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt index 55783f7db1f..6d37d4351fa 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt @@ -52,15 +52,8 @@ class Distribution(val targetManager: TargetManager, val dependenciesDir = DependencyProcessor.defaultDependenciesRoot.absolutePath val targetProperties = KonanProperties(target, properties, dependenciesDir) - val hostProperties = if (target == host) { - targetProperties - } else { - KonanProperties(host, properties, dependenciesDir) - } - val dependencies = targetProperties.dependencies - val llvmHome = hostProperties.absoluteLlvmHome - val hostSysRoot = hostProperties.absoluteTargetSysRoot + val llvmHome = targetProperties.absoluteLlvmHome val llvmBin = "$llvmHome/bin" val llvmLib = "$llvmHome/lib" val llvmLto = "$llvmBin/llvm-lto" diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 65cac5c5610..89cecbec984 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -54,7 +54,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration private fun Distribution.prepareDependencies(checkDependencies: Boolean) { if (checkDependencies) { - DependencyProcessor(java.io.File(dependenciesDir), targetProperties).run() + targetProperties.downloadDependencies() } } diff --git a/backend.native/konan.properties b/backend.native/konan.properties index 8d3a70a22fe..15d9029b30e 100644 --- a/backend.native/konan.properties +++ b/backend.native/konan.properties @@ -17,6 +17,9 @@ # TODO: Do we need a $variable substitution mechanism here? dependenciesUrl = https://download.jetbrains.com/kotlin/native +# In order of preference: +dependencyProfiles = default alt + # Don't check dependencies on a server. # If true, dependency downloader will throw an exception if a dependency isn't found in konan.home. airplaneMode = false diff --git a/build.gradle b/build.gradle index a0a8e2405fa..a7505d88814 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,16 @@ ext { konanPropertiesFile = project(':backend.native').file('konan.properties') konanProperties = PropertiesKt.loadProperties(konanPropertiesFile.absolutePath) + String DEPENDENCY_PROFILES_KEY = "dependencyProfiles" + def dependencyProfiles = konanProperties.getProperty(DEPENDENCY_PROFILES_KEY) + if (dependencyProfiles != "default alt") + throw new Error("unexpected $DEPENDENCY_PROFILES_KEY value: expected 'default alt', got '$dependencyProfiles'") + + // Force build to use only 'default' profile: + konanProperties.setProperty(DEPENDENCY_PROFILES_KEY, "default") + // TODO: it actually affects only resolution made in :dependencies, + // that's why we assume that 'default' profile comes first (and check this above). + distDir = file('dist') dependenciesDir = DependencyProcessor.defaultDependenciesRoot clangManager = new ClangManager(konanProperties, dependenciesDir.absolutePath) diff --git a/dependencies/build.gradle b/dependencies/build.gradle index 9bfd696b840..ea042ad5645 100644 --- a/dependencies/build.gradle +++ b/dependencies/build.gradle @@ -139,7 +139,7 @@ task update_kotlin_compiler(type: DefaultTask) { abstract class NativeDep extends DefaultTask { protected final String hostSystem = TargetManager.longerSystemName(); - String baseUrl = "https://jetbrains.bintray.com/kotlin-native-dependencies" + static final String baseUrl = "https://jetbrains.bintray.com/kotlin-native-dependencies" @Input abstract String getFileName() @@ -254,12 +254,19 @@ TargetManager.enabled.each { target -> } } + // Also resolves all dependencies: + final DependencyProcessor dependencyProcessor = new DependencyProcessor( + project.rootProject.ext.dependenciesDir, + rootProject.ext.konanProperties, + konanProperties.dependencies, + HelperNativeDep.baseUrl, + false + ) + DependencyKind.values().each { kind -> def dir = kind.getDirectory(konanProperties) if (dir != null) { - def path = rootProject.ext.dependenciesDir.toPath() - .resolve(dir).toFile() - .canonicalPath + String path = dependencyProcessor.resolveRelative(dir).canonicalPath rootProject.ext.set(kind.getPropertyName(target), path) } } diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanProperties.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanProperties.kt index 950a4289a5d..455722f5a73 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanProperties.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanProperties.kt @@ -18,9 +18,14 @@ package org.jetbrains.kotlin.konan.properties import org.jetbrains.kotlin.konan.file.* import org.jetbrains.kotlin.konan.target.* +import org.jetbrains.kotlin.konan.util.DependencyProcessor class KonanProperties(val target: KonanTarget, val properties: Properties, val baseDir: String? = null) { + fun downloadDependencies() { + dependencyProcessor!!.run() + } + fun targetString(key: String): String? = properties.targetString(key, target) fun targetList(key: String): List @@ -34,6 +39,8 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b fun hostTargetList(key: String): List = properties.hostTargetList(key, target) + val llvmHome get() = hostString("llvmHome") + // TODO: Delegate to a map? val llvmLtoNooptFlags get() = targetList("llvmLtoNooptFlags") val llvmLtoOptFlags get() = targetList("llvmLtoOptFlags") @@ -51,12 +58,12 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b val libffiDir get() = targetString("libffiDir") val gccToolchain get() = targetString("gccToolchain") val targetArg get() = targetString("quadruple") - val llvmHome get() = targetString("llvmHome") // Notice: these ones are host-target. val targetToolchain get() = hostTargetString("targetToolchain") val dependencies get() = hostTargetList("dependencies") - fun absolute(value: String?) = "${baseDir!!}/${value!!}" + private fun absolute(value: String?): String = + dependencyProcessor!!.resolveRelative(value!!).absolutePath val absoluteTargetSysRoot get() = absolute(targetSysRoot) val absoluteTargetToolchain get() = absolute(targetToolchain) @@ -75,4 +82,7 @@ class KonanProperties(val target: KonanTarget, val properties: Properties, val b } val osVersionMin: String? get() = targetString("osVersionMin") + + private val dependencyProcessor = baseDir?.let { DependencyProcessor(java.io.File(it), this) } + } diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt index 00eee2c0294..4bf8a1c7130 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt @@ -19,6 +19,7 @@ package org.jetbrains.kotlin.konan.util import org.jetbrains.kotlin.konan.file.use import org.jetbrains.kotlin.konan.properties.KonanProperties import org.jetbrains.kotlin.konan.properties.Properties +import org.jetbrains.kotlin.konan.properties.propertyList import java.io.File import java.io.FileNotFoundException import java.io.RandomAccessFile @@ -45,6 +46,26 @@ private val Properties.downloadingAttemptIntervalMs : Long private val Properties.homeDependencyCache : String get() = getProperty("homeDependencyCache") ?: DependencyProcessor.DEFAULT_HOME_DEPENDENCY_CACHE +private fun Properties.findCandidates(dependencies: List): Map> { + val dependencyProfiles = this.propertyList("dependencyProfiles") + return dependencies.map { dependency -> + dependency to dependencyProfiles.flatMap { profile -> + val candidateSpecs = propertyList("$dependency.$profile") + if (profile == "default" && candidateSpecs.isEmpty()) { + listOf(DependencySource.Remote.Public) + } else { + candidateSpecs.map { candidateSpec -> + when (candidateSpec) { + "remote:public" -> DependencySource.Remote.Public + "remote:internal" -> DependencySource.Remote.Internal + else -> DependencySource.Local(File(candidateSpec)) + } + } + } + } + }.toMap() +} + private val KonanProperties.dependenciesUrl : String get() = properties.dependenciesUrl private val KonanProperties.airplaneMode : Boolean get() = properties.airplaneMode @@ -52,6 +73,17 @@ private val KonanProperties.downloadingAttempts : Int get() = properti private val KonanProperties.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs private val KonanProperties.homeDependencyCache : String get() = properties.homeDependencyCache +private val internalDependenciesUrl = "http://repo.labs.intellij.net/kotlin-native" + +sealed class DependencySource { + data class Local(val path: File) : DependencySource() + + sealed class Remote : DependencySource() { + object Public : Remote() + object Internal : Remote() + } +} + // TODO: Try to use some dependency management system (Ivy?) // TODO: Maybe rename. /** @@ -60,13 +92,13 @@ private val KonanProperties.homeDependencyCache : String get() = properti */ class DependencyProcessor(dependenciesRoot: File, val dependenciesUrl: String, - val dependencies: Collection, + dependencyToCandidates: Map>, homeDependencyCache: String = DEFAULT_HOME_DEPENDENCY_CACHE, val airplaneMode: Boolean = false, maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS, attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS, customProgressCallback: ProgressCallback? = null, - val keepUnstable:Boolean = true) { + val keepUnstable: Boolean = true) { val dependenciesDirectory = dependenciesRoot.apply { mkdirs() } val cacheDirectory = System.getProperty("user.home")?.let { @@ -101,7 +133,7 @@ class DependencyProcessor(dependenciesRoot: File, keepUnstable:Boolean = true) : this( dependenciesRoot, dependenciesUrl, - dependencies, + dependencyToCandidates = properties.findCandidates(dependencies), airplaneMode = properties.airplaneMode, maxAttempts = properties.downloadingAttempts, attemptIntervalMs = properties.downloadingAttemptIntervalMs, @@ -137,13 +169,13 @@ class DependencyProcessor(dependenciesRoot: File, } } - private fun processDependency(dependency: String) { + private fun downloadDependency(dependency: String, baseUrl: String) { val depDir = File(dependenciesDirectory, dependency) val depName = depDir.name val fileName = "$depName.$archiveExtension" val archive = cacheDirectory.resolve(fileName) - val url = URL("$dependenciesUrl/$fileName") + val url = URL("$baseUrl/$fileName") val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted") if (extractedDependencies.contains(depName) && @@ -190,10 +222,57 @@ class DependencyProcessor(dependenciesRoot: File, get() = Paths.get(System.getProperty("user.home")).resolve(".konan/dependencies").toFile() } + private fun internalServerIsAvailable(): Boolean = System.getenv("KONAN_USE_INTERNAL_SERVER") != null + + private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) -> + val candidate = candidates.asSequence().mapNotNull { candidate -> + when (candidate) { + is DependencySource.Local -> candidate.takeIf { it.path.exists() } + DependencySource.Remote.Public -> candidate + DependencySource.Remote.Internal -> candidate.takeIf { internalServerIsAvailable() } + } + }.firstOrNull() + + candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}") + + dependency to candidate + }.toMap() + + private fun resolveDependency(dependency: String): File { + val candidate = resolvedDependencies[dependency] + return when (candidate) { + is DependencySource.Local -> candidate.path + is DependencySource.Remote -> File(dependenciesDirectory, dependency) + null -> error("$dependency not declared as dependency") + } + } + + fun resolveRelative(relative: String): File { + val path = Paths.get(relative) + if (path.isAbsolute) error("not a relative path: $relative") + + val dependency = path.first().toString() + return resolveDependency(dependency).let { + if (path.nameCount > 1) { + it.toPath().resolve(path.subpath(1, path.nameCount)).toFile() + } else { + it + } + } + } + fun run() = lock.withLock { RandomAccessFile(lockFile, "rw").channel.lock().use { - dependencies.forEach { - processDependency(it) + resolvedDependencies.forEach { (dependency, candidate) -> + val baseUrl = when (candidate) { + is DependencySource.Local -> null + DependencySource.Remote.Public -> dependenciesUrl + DependencySource.Remote.Internal -> internalDependenciesUrl + } + // TODO: consider using different caches for different remotes. + if (baseUrl != null) { + downloadDependency(dependency, baseUrl) + } } } } diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/tasks/KonanCompilerDownloadTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/tasks/KonanCompilerDownloadTask.kt index 33bd6c682ef..38ebc540db3 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/tasks/KonanCompilerDownloadTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/tasks/KonanCompilerDownloadTask.kt @@ -22,6 +22,7 @@ import org.gradle.api.tasks.Internal import org.gradle.api.tasks.TaskAction import org.jetbrains.kotlin.gradle.plugin.* import org.jetbrains.kotlin.konan.util.DependencyProcessor +import org.jetbrains.kotlin.konan.util.DependencySource import java.io.File import java.io.IOException @@ -60,7 +61,11 @@ open class KonanCompilerDownloadTask : DefaultTask() { } val konanCompiler = project.konanCompilerName() logger.info("Downloading Kotlin/Native compiler from $downloadUrlDirectory/$konanCompiler into $KONAN_PARENT_DIR") - DependencyProcessor(File(KONAN_PARENT_DIR), downloadUrlDirectory, listOf(konanCompiler)).run() + DependencyProcessor( + File(KONAN_PARENT_DIR), + downloadUrlDirectory, + mapOf(konanCompiler to listOf(DependencySource.Remote.Public)) + ).run() } catch (e: IOException) { throw GradleScriptException("Cannot download Kotlin/Native compiler", e) }