diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index 306306273d6..94d0f961539 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -187,7 +187,7 @@ private fun runCmd(command: Array, workDir: File, verbose: Boolean = fal private fun maybeExecuteHelper(dependenciesRoot: String, properties: Properties, dependencies: List) { try { - val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin + val kClass = Class.forName("org.jetbrains.kotlin.konan.util.Helper0").kotlin @Suppress("UNCHECKED_CAST") val ctor = kClass.constructors.single() as KFunction val result = ctor.call(dependenciesRoot, properties, dependencies) diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index c63353c4d5b..ec980257735 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -33,28 +33,9 @@ import org.jetbrains.kotlin.config.addKotlinSourceRoots import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.TargetManager import org.jetbrains.kotlin.utils.KotlinPaths +import java.io.File import kotlin.reflect.KFunction -// TODO: Don't use reflection? -private fun maybeExecuteHelper(targetName: String) { - try { - val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin - @Suppress("UNCHECKED_CAST") - val ctor = kClass.constructors.single() as KFunction - val distribution = Distribution(TargetManager(targetName)) - val result = ctor.call( - distribution.dependenciesDir, - distribution.properties, - distribution.dependencies - ) - result.run() - } catch (notFound: ClassNotFoundException) { - // Just ignore, no helper. - } catch (e: Throwable) { - throw IllegalStateException("Cannot download dependencies.", e) - } -} - class K2Native : CLICompiler() { override fun doExecute(@NotNull arguments: K2NativeCompilerArguments, @NotNull configuration: CompilerConfiguration, @NotNull rootDisposable: Disposable, @Nullable paths: KotlinPaths?): ExitCode { @@ -85,7 +66,6 @@ class K2Native : CLICompiler() { return this?.asList() ?: listOf() } - // It is executed before doExecute(). override fun setupPlatformSpecificArgumentsAndServices( configuration: CompilerConfiguration, @@ -152,8 +132,6 @@ class K2Native : CLICompiler() { put(ENABLE_ASSERTIONS, arguments.enableAssertions) } } - - maybeExecuteHelper(arguments.target ?: "host") } override fun createArguments(): K2NativeCompilerArguments { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt index ae16f3a7436..134968fcfd2 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt @@ -18,11 +18,11 @@ // Example startup helper for Kotlin N compiler. Compile to JAR and add it to // Kotlin N classpath - it will get loaded and executed automatically. // -package org.jetbrains.kotlin.konan +package org.jetbrains.kotlin.konan.util import java.util.* -// Name it CompilerHelper0 so that it get loaded. +// Name it Helper0 so that it get loaded. class Helper0Disabled(val dependenciesDir: String, val properties: Properties, val dependencies: List) : Runnable { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index c286af877e9..0376d03ac0b 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -33,6 +33,7 @@ import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.* import org.jetbrains.kotlin.konan.target.TargetManager.* import org.jetbrains.kotlin.konan.target.CompilerOutputKind.* +import org.jetbrains.kotlin.konan.util.DependencyProcessor class KonanConfig(val project: Project, val configuration: CompilerConfiguration) { @@ -47,9 +48,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration error("Target $target is not available on the ${TargetManager.host} host") } } + + private fun Distribution.prepareDependencies() { + DependencyProcessor(java.io.File(dependenciesDir), targetProperties).run() + } + internal val distribution = Distribution(targetManager, - configuration.get(KonanConfigKeys.PROPERTY_FILE), - configuration.get(KonanConfigKeys.RUNTIME_FILE)) + configuration.get(KonanConfigKeys.PROPERTY_FILE), + configuration.get(KonanConfigKeys.RUNTIME_FILE)).apply { + prepareDependencies() + } private val produce = configuration.get(KonanConfigKeys.PRODUCE)!! private val suffix = produce.suffix(targetManager.target) diff --git a/backend.native/konan.properties b/backend.native/konan.properties index 42d3d465142..3141085820b 100644 --- a/backend.native/konan.properties +++ b/backend.native/konan.properties @@ -22,7 +22,8 @@ dependenciesUrl = http://download.jetbrains.com/kotlin/native airplaneMode = false downloadingAttempts = 10 -downloadingAttemptPauseMs = 3000 +downloadingAttemptIntervalMs = 3000 +homeDependencyCache = .konan/cache llvmDebugOptFlags = -O0 diff --git a/build.gradle b/build.gradle index b98809a2001..1e711479514 100644 --- a/build.gradle +++ b/build.gradle @@ -143,7 +143,6 @@ task list_dist(dependsOn: "listDist") task distCompiler(type: Copy) { dependsOn ':backend.native:jars' - dependsOn ':tools:helpers:jar' dependsOn ':utilities:jar' dependsOn ':klib:jar' dependsOn ':shared:jar' @@ -212,11 +211,6 @@ task distCompiler(type: Copy) { from(konanPropertiesFile) { into('konan') } - - // TODO: check if we use native libraries copied to dist (see above) or not. - from(project(':tools:helpers').file('build/libs')) { - into('konan/lib') - } } task listDist(type: Exec) { diff --git a/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 006a9711ce2..c0a44db40e6 100644 --- a/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -100,8 +100,6 @@ abstract class KonanTest extends JavaExec { try { main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' classpath = project.configurations.cli_bc - // TODO: Remove helpers classpath when downloading is moved into 'shared' project. - classpath += project.project(":tools:helpers").sourceSets.main.runtimeClasspath enableAssertions = true args = ["-output", output, *filesToCompile, diff --git a/cmd/run_konan b/cmd/run_konan index fb9a776b348..c2ad971883f 100755 --- a/cmd/run_konan +++ b/cmd/run_konan @@ -79,11 +79,10 @@ KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar" STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar" INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar" INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar" -HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar" SHARED_JAR="${KONAN_HOME}/konan/lib/shared.jar" KLIB_JAR="${KONAN_HOME}/konan/lib/klib.jar" UTILITIES_JAR="${KONAN_HOME}/konan/lib/utilities.jar" -KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$HELPERS_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR" +KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR" TOOL_CLASS=org.jetbrains.kotlin.cli.utilities.MainKt LIBCLANG_DISABLE_CRASH_RECOVERY=1 \ diff --git a/dependencies/build.gradle b/dependencies/build.gradle index e92939772e1..4d0cb3120ea 100644 --- a/dependencies/build.gradle +++ b/dependencies/build.gradle @@ -16,7 +16,12 @@ import groovy.transform.stc.ClosureParams +import groovy.transform.stc.ClosureSignatureHint +import groovy.transform.stc.FirstParam +import groovy.transform.stc.FromString import org.jetbrains.kotlin.konan.target.* +import org.jetbrains.kotlin.konan.util.DependencyProcessor + import static org.jetbrains.kotlin.konan.target.KonanTarget.* import org.jetbrains.kotlin.konan.properties.KonanProperties import org.jetbrains.kotlin.konan.properties.* @@ -166,23 +171,11 @@ class TgzNativeDep extends NativeDep { class HelperNativeDep extends TgzNativeDep { - int maxAttempts = Integer.parseInt( - project.rootProject.ext.konanProperties['downloadingAttempts'].toString()) - long attemptPauseMs = Long.parseLong( - project.rootProject.ext.konanProperties['downloadingAttemptPauseMs'].toString()) - - public HelperNativeDep() { - dependsOn(':tools:helpers:jar') - } + Properties konanProperties = new Properties() @TaskAction public void downloadAndExtract() { - project.javaexec { - main = "org.jetbrains.kotlin.konan.MainKt" - classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime") - classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime").artifacts.files - args baseOutDir.canonicalPath, baseUrl, maxAttempts, attemptPauseMs, baseName - } + new DependencyProcessor(baseOutDir, konanProperties, [baseName], baseUrl).run() } } @@ -195,8 +188,8 @@ enum DependencyKind { DependencyKind(String name, - @ClosureParams(KonanProperties.class) Closure dirGetter, - @ClosureParams(KonanTarget.class) Closure propertyNameGetter = + @ClosureParams(value = FromString.class, options = "KonanProperties") Closure dirGetter, + @ClosureParams(value = FromString.class, options = "KonanTarget") Closure propertyNameGetter = {target -> "${target.userName}${name.capitalize()}Dir"}) { this.name = name this.dirGetter = dirGetter @@ -225,8 +218,11 @@ TargetManager.enabled.each { target -> ) konanProperties.dependencies.each { dependency -> - task "${target.userName}-${dependency}"(type: HelperNativeDep) { - baseName = dependency + if (tasks.findByName(dependency) == null) { + task "${dependency}"(type: HelperNativeDep) { + baseName = dependency + it.konanProperties = rootProject.ext.konanProperties + } } } diff --git a/settings.gradle b/settings.gradle index 6eb7bc49fde..e06e588e9dd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,6 +26,5 @@ include ':runtime' include ':common' include ':backend.native:tests' include ':shared' -include ':tools:helpers' include ':tools:kotlin-native-gradle-plugin' include ':utilities' \ No newline at end of file diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/NoJavaUtil.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt similarity index 94% rename from shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/NoJavaUtil.kt rename to shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt index ebb35eeaa98..d599c774b6c 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/NoJavaUtil.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt @@ -182,7 +182,7 @@ fun Path.recursiveCopyTo(destPath: Path) { val relative = sourcePath.relativize(oldPath) val destFs = destPath.getFileSystem() // We are copying files between file systems, - // so pass the relative path through the Sting. + // so pass the relative path through the String. val newPath = destFs.getPath(destPath.toString(), relative.toString()) // File systems don't allow replacing an existing root. @@ -195,7 +195,7 @@ fun Path.recursiveCopyTo(destPath: Path) { fun bufferedReader(errorStream: InputStream?) = BufferedReader(InputStreamReader(errorStream)) // stdlib `use` function adapted for AutoCloseable. -private inline fun T.use(block: (T) -> R): R { +inline fun T.use(block: (T) -> R): R { var closed = false try { return block(this) @@ -212,3 +212,11 @@ private inline fun T.use(block: (T) -> R): R { } } } + +fun Path.unzipTo(directory: Path) { + val zipUri = URI.create("jar:" + this.toUri()) + FileSystems.newFileSystem(zipUri, emptyMap(), null).use { zipfs -> + val zipPath = zipfs.getPath("/") + zipPath.recursiveCopyTo(directory) + } +} diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt new file mode 100644 index 00000000000..03b11c01e1a --- /dev/null +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt @@ -0,0 +1,115 @@ +package org.jetbrains.kotlin.konan.util + +import java.io.EOFException +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.net.URL +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +class DependencyDownloader( + var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS, + var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS) { + + enum class ReplacingMode { + /** Redownload the file and replace the existing one. */ + REPLACE, + /** Throw FileAlreadyExistsException */ + THROW, + /** Don't download the file and return the existing one*/ + RETURN_EXISTING + } + + /** Performs an attempt to download a specified file into the specified location */ + fun tryDownload(url: URL, dstFile: File) { + val connection = url.openConnection() + val totalBytes = connection.contentLengthLong + var currentBytes = 0L + + try { + url.openStream().use { from -> + FileOutputStream(dstFile, false).use { to -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var read = from.read(buffer) + while (read != -1) { + to.write(buffer, 0, read) + currentBytes += read + updateProgressMsg(url.toString(), currentBytes, totalBytes) + read = from.read(buffer) + } + if (currentBytes != totalBytes) { + throw EOFException("The stream closed before end of downloading.") + } + } + } + } catch (e: Throwable) { + dstFile.delete() + throw e + } + } + + /** Downloads a file from [source] url to [destination]. Returns [destination]. */ + fun download(source: URL, + destination: File, + replace: ReplacingMode = ReplacingMode.RETURN_EXISTING): File { + + if (destination.exists()) { + @Suppress("NON_EXHAUSTIVE_WHEN") + when (replace) { + ReplacingMode.RETURN_EXISTING -> return destination + ReplacingMode.THROW -> throw FileAlreadyExistsException(destination) + // TODO: What if dst is a directory? + } + } + // TODO: resuming must be here + val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX") + + var attempt = 1 + var waitTime = 0L + while (true) { + try { + tryDownload(source, tmpFile) + break + } catch (e: IOException) { + if (attempt >= maxAttempts) { + throw e + } + attempt++ + waitTime += attemptIntervalMs + println("Cannot download a dependency: $e\n" + + "Wait ${waitTime.toDouble() / 1000} sec and try again (attempt: $attempt/$maxAttempts).") + // TODO: Wait better + Thread.sleep(waitTime) + } + } + + Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING) + println("Done.") + return destination + } + + private val Long.humanReadable: String + get() { + if (this < 0) { + return "-" + } + if (this < 1024) { + return "$this bytes" + } + val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt() + val prefix = "kMGTPE"[exp-1] + return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix) + } + + private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) { + print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ") + } + + companion object { + const val DEFAULT_MAX_ATTEMPTS = 10 + const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L + + const val TMP_SUFFIX = "part" + } +} diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt new file mode 100644 index 00000000000..7af9fab39ec --- /dev/null +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt @@ -0,0 +1,35 @@ +package org.jetbrains.kotlin.konan.util + +import org.jetbrains.kotlin.konan.file.unzipTo +import java.io.File + + +class DependencyExtractor { + internal val useZip = System.getProperty("os.name").startsWith("Windows") + + internal val archiveExtension = if (useZip) { + "zip" + } else { + "tar.gz" + } + + private fun extractTarGz(tarGz: File, targetDirectory: File) { + println("Extract dependency: ${tarGz.canonicalPath} in ${targetDirectory.canonicalPath}") + val tarProcess = ProcessBuilder().apply { + command("tar", "-xzf", tarGz.canonicalPath) + directory(targetDirectory) + }.start() + tarProcess.waitFor() + if (tarProcess.exitValue() != 0) { + throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}") + } + } + + fun extract(archive: File, targetDirectory: File) { + if (useZip) { + archive.toPath().unzipTo(targetDirectory.toPath()) + } else { + extractTarGz(archive, targetDirectory) + } + } +} \ No newline at end of file diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt new file mode 100644 index 00000000000..35fcf650209 --- /dev/null +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt @@ -0,0 +1,173 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.konan.util + +import org.jetbrains.kotlin.konan.file.use +import org.jetbrains.kotlin.konan.properties.KonanProperties +import org.jetbrains.kotlin.konan.properties.Properties +import java.io.File +import java.io.FileNotFoundException +import java.io.RandomAccessFile +import java.net.URL +import java.nio.file.Paths +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +private val Properties.dependenciesUrl : String + get() = getProperty("dependenciesUrl") + ?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl") + +private val Properties.airplaneMode : Boolean + get() = getProperty("airplaneMode")?.toBoolean() ?: false + +private val Properties.downloadingAttempts : Int + get() = getProperty("downloadingAttempts")?.toInt() + ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS + +private val Properties.downloadingAttemptIntervalMs : Long + get() = getProperty("downloadingAttemptPauseMs")?.toLong() + ?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS + +private val Properties.homeDependencyCache : String + get() = getProperty("homeDependencyCache") ?: DependencyProcessor.DEFAULT_HOME_DEPENDENCY_CACHE + + +private val KonanProperties.dependenciesUrl : String get() = properties.dependenciesUrl +private val KonanProperties.airplaneMode : Boolean get() = properties.airplaneMode +private val KonanProperties.downloadingAttempts : Int get() = properties.downloadingAttempts +private val KonanProperties.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs +private val KonanProperties.homeDependencyCache : String get() = properties.homeDependencyCache + +// TODO: Try to use some dependency management system (Ivy?) +// TODO: Maybe rename. +/** + * Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl]. + * If [airplaneMode] is true will throw a RuntimeException instead of downloading. + */ +class DependencyProcessor(dependenciesRoot: File, + val dependenciesUrl: String, + val dependencies: Collection, + homeDependencyCache: String = DEFAULT_HOME_DEPENDENCY_CACHE, + val airplaneMode: Boolean = false, + maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS, + attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS) { + + val dependenciesDirectory = dependenciesRoot.apply { mkdirs() } + val cacheDirectory = System.getProperty("user.home")?.let { + Paths.get(it).resolve(homeDependencyCache).toFile().apply { mkdirs() } + } ?: dependenciesRoot + + val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() } + + private var isInfoShown = false + + // TOOO: Rename pause -> interval + private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs) + private val extractor = DependencyExtractor() + + private val archiveExtension get() = extractor.archiveExtension + + constructor(dependenciesRoot: File, + properties: KonanProperties, + dependenciesUrl: String = properties.dependenciesUrl) : this( + dependenciesRoot, + properties.properties, + properties.dependencies, + dependenciesUrl) + + constructor(dependenciesRoot: File, + properties: Properties, + dependencies: List, + dependenciesUrl: String = properties.dependenciesUrl) : this( + dependenciesRoot, + dependenciesUrl, + dependencies, + airplaneMode = properties.airplaneMode, + maxAttempts = properties.downloadingAttempts, + attemptIntervalMs = properties.downloadingAttemptIntervalMs) + + + class DependencyFile(directory: File, fileName: String) { + val file = File(directory, fileName).apply { createNewFile() } + private val dependencies = file.readLines().toMutableSet() + + fun contains(dependency: String) = dependencies.contains(dependency) + fun add(dependency: String) = dependencies.add(dependency) + fun addWithSave(dependency: String) { + add(dependency) + save() + } + + fun save() { + val writer = file.writer() + writer.use { + dependencies.forEach { + writer.write(it) + writer.write("\n") + } + } + } + } + + private fun processDependency(dependency: String) { + val depDir = File(dependenciesDirectory, dependency) + val depName = depDir.name + + val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted") + if (extractedDependencies.contains(depName) && + depDir.exists() && + depDir.isDirectory && + depDir.list().isNotEmpty()) { + return + } + + if (!isInfoShown) { + println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.") + isInfoShown = true + } + val fileName = "$depName.$archiveExtension" + val archive = cacheDirectory.resolve(fileName) + val url = URL("$dependenciesUrl/$fileName") + + if (!archive.exists()) { + if (airplaneMode) { + throw FileNotFoundException(""" + Cannot find a dependency locally: $dependency. + Set `airplaneMode = false` in konan.properties to download it. + """.trimIndent()) + } else { + downloader.download(url, archive) + } + } + extractor.extract(archive, dependenciesDirectory) + extractedDependencies.addWithSave(depName) + } + + companion object { + val lock = ReentrantLock() + + const val DEFAULT_HOME_DEPENDENCY_CACHE = ".konan/cache" + } + + fun run() = lock.withLock { + RandomAccessFile(lockFile, "rw").channel.lock().use { + dependencies.forEach { + processDependency(it) + } + } + } +} diff --git a/tools/helpers/src/main/kotlin/Helper0.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/Helper0.kt similarity index 62% rename from tools/helpers/src/main/kotlin/Helper0.kt rename to shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/Helper0.kt index f2b859d8351..ec2169040ae 100644 --- a/tools/helpers/src/main/kotlin/Helper0.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/Helper0.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.jetbrains.kotlin.konan +package org.jetbrains.kotlin.konan.util import java.io.File import java.util.* @@ -24,15 +24,10 @@ class Helper0(val dependenciesDir: String, val dependencies: List): Runnable { override fun run() { - DependencyDownloader( - File(dependenciesDir), - properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native"), + DependencyProcessor(File(dependenciesDir), + properties, dependencies, - airplaneMode = properties.getProperty("airplaneMode")?.toBoolean() ?: false, - maxAttempts = properties.getProperty("downloadingAttempts")?.toInt() - ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS, - attemptPauseMs = properties.getProperty("downloadingAttemptPauseMs")?.toLong() - ?: DependencyDownloader.DEFAULT_ATTEMPT_PAUSE_MS + properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native") ).run() } } \ No newline at end of file diff --git a/tools/helpers/build.gradle b/tools/helpers/build.gradle deleted file mode 100644 index 568ba178dec..00000000000 --- a/tools/helpers/build.gradle +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.jetbrains.kotlin.NativeInteropPlugin -import org.jetbrains.kotlin.RunInteropKonanTest -import org.jetbrains.kotlin.RunKonanTest -import org.jetbrains.kotlin.TestFailedException - -import java.util.concurrent.atomic.AtomicInteger - -buildscript { - apply from: "$rootDir/gradle/kotlinGradlePlugin.gradle" -} - -// Copied from backend.native project. ===== -repositories { - maven { url kotlinCompilerRepo } -} - -configurations.all { - // kotlin-compiler module includes Kotlin runtime bundled; - // make Gradle aware of this to avoid multiple Kotlin runtimes in classpath: - resolutionStrategy.dependencySubstitution { - substitute module('org.jetbrains.kotlin:kotlin-runtime') with module(kotlinCompilerModule) - substitute module('org.jetbrains.kotlin:kotlin-stdlib') with module(kotlinCompilerModule) - substitute module('org.jetbrains.kotlin:kotlin-reflect') with module(kotlinCompilerModule) - } -} -// ========================================= - -apply plugin: 'kotlin' -apply plugin: NativeInteropPlugin - -configurations { - cli_bc.extendsFrom compile -} - -dependencies { - compile "org.jetbrains.kotlin:kotlin-runtime:$kotlin_version" - // We need this configuration to test the helpers. - cli_bc jar.outputs.files - cli_bc project(path: ':backend.native', configuration: 'cli_bc') -} - -sourceSets { - testOutputLocal { - output.dir(rootProject.file("${findProject(":backend.native:tests").testOutputRoot}/tools/helpers")) - } -} - -def testDependencies = rootProject.file('dist/dependencies') - -void prepareDependenciesDir(File testDependencies) { - project.delete testDependencies - testDependencies.mkdirs() - // Delete all sysroots from the dependencies cache to tests their downloading. - project.delete(project.fileTree("${System.getProperty("user.home")}/.konan/cache/") { - include "*sysroot*.tar.gz" - }) -} - -kotlinNativeInterop { - sysstat { - pkg 'sysstat' - headers 'sys/stat.h' - flavor 'native' - } -} - -project.convention.plugins.remoteExec = new org.jetbrains.kotlin.ExecRemote(project) - -task testInteropHelper(type: RunInteropKonanTest) { - dependsOn jar - dependsOn ':dist' - - dependenciesDir = testDependencies - goldValue = "0\n0\n" - source = "testData/interop.kt" - interop = 'sysstat' - - doFirst { - prepareDependenciesDir(testDependencies) - project.delete(project.file(buildExePath()).parent) - } - //Interop gradle plugin doesn't save its work log so we cannot use it to check if dependencies were downloaded or not. -} - -task testCompilerHelper(type: RunKonanTest) { - dependsOn jar - dependsOn ':dist' - - dependenciesDir = testDependencies - source = "testData/main.kt" - - doFirst { - prepareDependenciesDir(testDependencies) - project.delete(project.file(buildExePath()).parent) - } - - doLast { - if (!file("${buildExePath()}.compilation.log").text.contains("Extract dependency") || - !file("${buildExePath()}.compilation.log").text.contains("Download dependency")) { - throw new TestFailedException("Compilation log contains no Extract/Download messages") - } - } -} - -// Simple test for parallel compiler execution -task testParallel(type: DefaultTask) { - dependsOn jar - dependsOn ':dist' - - for (int i=0; i<4; i++) { - task("runParallel$i", type: RunKonanTest) { - dependenciesDir = testDependencies - source = "testData/main.kt" - } - } - - doFirst { - prepareDependenciesDir(testDependencies) - } - - doLast { - List threads = new ArrayList<>() - def downloadCnt = new AtomicInteger(0) - def extractCnt = new AtomicInteger(0) - tasks.withType(RunKonanTest).matching { it.name.startsWith("runParallel") }.each { task -> - threads.add(Thread.start { - task.executeTest() - // The helper prints messages when it is downloading. - // We check that there is only one downloading using compilation logs. - if (file("${task.buildExePath()}.compilation.log").text.contains("Extract dependency")) { - extractCnt.incrementAndGet() - } - if (file("${task.buildExePath()}.compilation.log").text.contains("Download dependency")) { - downloadCnt.incrementAndGet() - } - }) - } - threads.each { it.join() } - if (downloadCnt.intValue() != 1 || extractCnt.intValue() != 1) { - throw new TestFailedException("Actual downloads: ${downloadCnt.intValue()}, expected: 1\n" + - "Actial extractions: ${extractCnt.intValue()}, expected: 1") - } - } -} - -task testHelpers { - dependsOn testInteropHelper - dependsOn testCompilerHelper - dependsOn testParallel -} diff --git a/tools/helpers/src/main/kotlin/DependencyDownloader.kt b/tools/helpers/src/main/kotlin/DependencyDownloader.kt deleted file mode 100644 index b367d93fdba..00000000000 --- a/tools/helpers/src/main/kotlin/DependencyDownloader.kt +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.konan - -import java.io.* -import java.net.URL -import java.nio.file.Files -import java.nio.file.StandardCopyOption -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.thread -import kotlin.concurrent.withLock - -// TODO: Try to use some dependency management system (Ivy?) -/** - * Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl]. - * If [airplaneMode] is true will throw a RuntimeException instead of downloading. - */ -class DependencyDownloader(dependenciesRoot: File, - val dependenciesUrl: String, - val dependencies: List, - val airplaneMode: Boolean = false, - val maxAttempts: Int = DEFAULT_MAX_ATTEMPTS, - val attemptPauseMs: Long = DEFAULT_ATTEMPT_PAUSE_MS) { - - val dependenciesDirectory = dependenciesRoot.apply { mkdirs() } - val cacheDirectory = System.getProperty("user.home")?.let { - File("$it/.konan/cache").apply { mkdirs() } - } ?: dependenciesRoot - - val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() } - - var isInfoShown = false - - class DependencyFile(directory: File, fileName: String) { - val file = File(directory, fileName).apply { createNewFile() } - private val dependencies = file.readLines().toMutableSet() - - fun contains(dependency: String) = dependencies.contains(dependency) - fun add(dependency: String) = dependencies.add(dependency) - fun addWithSave(dependency: String) { - add(dependency) - save() - } - - fun save() { - val writer = file.writer() - writer.use { - dependencies.forEach { - writer.write(it) - writer.write("\n") - } - } - } - } - - private val useZip = System.getProperty("os.name").startsWith("Windows") - - private val archiveExtension = if (useZip) { - "zip" - } else { - "tar.gz" - } - - private fun processDependency(dependency: String) { - val depDir = File(dependenciesDirectory, dependency) - val depName = depDir.name - - val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted") - if (extractedDependencies.contains(depName) && - depDir.exists() && - depDir.isDirectory && - depDir.list().isNotEmpty()) { - return - } - - if (!isInfoShown) { - println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.") - isInfoShown = true - } - - val archive = File(cacheDirectory.canonicalPath, "$depName.$archiveExtension") - if (!archive.exists()) { - if (!airplaneMode) { - var attempt = 0 - while(true) { - try { - download(depName, archive) - break - } catch (e: IOException) { - if (attempt < maxAttempts) { - attempt++ - val pauseTime = attempt * attemptPauseMs - println("Downloading error: ${e.message}.\n" + - "Waiting ${pauseTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)") - Thread.sleep(pauseTime) - } else { throw e } - } - } - } else { - throw RuntimeException(""" - Cannot find a dependency locally: $dependency. - Set `airplaneMode = false` in konan.properties to download it. - """.trimIndent()) - } - } - extract(archive, dependenciesDirectory) - extractedDependencies.addWithSave(depName) - } - - private fun extract(archive: File, target: File) { - if (useZip) { - archive.toPath().unzipTo(target.toPath()) - } else { - extractTarGz(archive, target) - } - } - - private fun extractTarGz(tarGz: File, target: File) { - println("Extract dependency: ${tarGz.canonicalPath} in ${target.canonicalPath}") - val tarProcess = ProcessBuilder().apply { - command("tar", "-xzf", "${tarGz.canonicalPath}") - directory(target) - }.start() - tarProcess.waitFor() - if (tarProcess.exitValue() != 0) { - throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}") - } - } - - private val Long.humanReadable: String - get() { - if (this < 0) { - return "-" - } - if (this < 1024) { - return "$this bytes" - } - val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt() - val prefix = "kMGTPE"[exp-1] - return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix) - } - - private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) { - print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ") - } - - private fun download(dependencyName: String, outputFile: File) { - val tmpFile = File("${outputFile.canonicalPath}.part") - val url = URL("$dependenciesUrl/$dependencyName.$archiveExtension") - val connection = url.openConnection() - val totalBytes = connection.contentLengthLong - - var currentBytes = 0L - var done = false - var downloadError: Throwable? = null - - thread { - try { - url.openStream().use { from -> - FileOutputStream(tmpFile, false).use { to -> - val buffer = ByteArray(DEFAULT_BUFFER_SIZE) - var read = from.read(buffer) - while (read != -1) { - to.write(buffer, 0, read) - currentBytes += read - read = from.read(buffer) - } - if (currentBytes != totalBytes) { - throw EOFException("The stream closed before end of downloading.") - } - } - } - } catch (e: Throwable) { - downloadError = e - } - done = true - } - - // TODO: Improve console logging - while (!done) { - Thread.sleep(1000) // We can use condition variable here. - updateProgressMsg(url.toString(), currentBytes, totalBytes) - } - println("Done.") - if (downloadError != null) { - tmpFile.delete() - throw IOException("Cannot download dependency: $url", downloadError) - } - Files.move( - tmpFile.toPath(), - outputFile.toPath(), - StandardCopyOption.REPLACE_EXISTING - ) - } - - // stdlib `use` function adapted for AutoCloseable. - private inline fun T.use(block: (T) -> R): R { - var closed = false - try { - return block(this) - } catch (e: Exception) { - closed = true - try { - this?.close() - } catch (closeException: Exception) { - } - throw e - } finally { - if (!closed) { - this?.close() - } - } - } - - companion object { - val lock = ReentrantLock() - - const val DEFAULT_MAX_ATTEMPTS = 10 - const val DEFAULT_ATTEMPT_PAUSE_MS = 3000L - } - - fun run() = lock.withLock { - RandomAccessFile(lockFile, "rw").channel.lock().use { - dependencies.forEach { - processDependency(it) - } - } - } -} diff --git a/tools/helpers/src/main/kotlin/Unzip.kt b/tools/helpers/src/main/kotlin/Unzip.kt deleted file mode 100644 index b0d85f6a92c..00000000000 --- a/tools/helpers/src/main/kotlin/Unzip.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.konan - -import java.net.URI -import java.nio.file.FileSystems -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption - -fun Path.unzipTo(directory: Path) { - val zipUri = URI.create("jar:" + this.toUri()) - FileSystems.newFileSystem(zipUri, emptyMap(), null).use { zipfs -> - val zipPath = zipfs.getPath("/") - zipPath.recursiveCopyTo(directory) - } -} - -fun Path.recursiveCopyTo(destPath: Path) { - val sourcePath = this - Files.walk(sourcePath).forEach { oldPath -> - - val relative = sourcePath.relativize(oldPath) - - // We are copying files between file systems, - // so pass the relative path through the Sting. - val newPath = destPath.resolve(relative.toString()) - - if (Files.isDirectory(oldPath)) { - Files.createDirectories(newPath) - // TODO: consider copying attributes. - } else { - Files.copy(oldPath, newPath, - StandardCopyOption.REPLACE_EXISTING) - } - } -} \ No newline at end of file diff --git a/tools/helpers/src/main/kotlin/main.kt b/tools/helpers/src/main/kotlin/main.kt deleted file mode 100644 index bfeb28a3875..00000000000 --- a/tools/helpers/src/main/kotlin/main.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.konan - -import java.io.File -import java.util.* - -// TODO: Add command line keys -// Args: dependencies directory, dependencies url, max attempts, attempt pause, dependencies list -fun main(args: Array) { - if (args.size < 5) { - System.exit(1) - } - val dependenciesDir = File(args[0]) - val dependenciesUrl = args[1] - val maxAttempts = args[2].toInt() - val attemptPause = args[3].toLong() - val dependencies = List(args.size - 4) { args[4 + it] } - DependencyDownloader(dependenciesDir, - dependenciesUrl, - dependencies, - maxAttempts = maxAttempts, - attemptPauseMs = attemptPause - ).run() -} diff --git a/tools/helpers/testData/interop.kt b/tools/helpers/testData/interop.kt deleted file mode 100644 index 2654255db69..00000000000 --- a/tools/helpers/testData/interop.kt +++ /dev/null @@ -1,9 +0,0 @@ -import sysstat.* -import kotlinx.cinterop.* - -fun main(args: Array) { - val statBuf = nativeHeap.alloc() - val res = stat("/", statBuf.ptr) - println(res) - println(statBuf.st_uid) -} \ No newline at end of file diff --git a/tools/helpers/testData/main.kt b/tools/helpers/testData/main.kt deleted file mode 100644 index 96b607ab67d..00000000000 --- a/tools/helpers/testData/main.kt +++ /dev/null @@ -1,3 +0,0 @@ -fun main(args: Array) { - println(42) -} \ No newline at end of file diff --git a/tools/kotlin-native-gradle-plugin/build.gradle b/tools/kotlin-native-gradle-plugin/build.gradle index 80019572fcd..54ecbbb02bd 100644 --- a/tools/kotlin-native-gradle-plugin/build.gradle +++ b/tools/kotlin-native-gradle-plugin/build.gradle @@ -47,11 +47,8 @@ repositories { mavenCentral() } -evaluationDependsOn(':tools:helpers') - dependencies { compile project(':shared') - compile project(':tools:helpers') compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" compile gradleApi() @@ -69,8 +66,6 @@ test { } jar { - dependsOn(':tools:helpers:jar') - from (rootProject.findProject(':tools:helpers').sourceSets.main.output) from (rootProject.findProject(':shared').sourceSets.main.output) } diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt index 9201c9d0e47..eb42b4f0ab9 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.gradle.plugin import org.gradle.api.DefaultTask import org.gradle.api.GradleScriptException import org.gradle.api.tasks.TaskAction -import org.jetbrains.kotlin.konan.DependencyDownloader +import org.jetbrains.kotlin.konan.util.DependencyProcessor import java.io.File open class KonanCompilerDownloadTask : DefaultTask() { @@ -53,7 +53,7 @@ open class KonanCompilerDownloadTask : DefaultTask() { try { val konanCompiler = project.konanCompilerName() logger.info("Downloading Kotlin/Native compiler from $DOWNLOAD_URL/$konanCompiler into $KONAN_PARENT_DIR") - DependencyDownloader(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run() + DependencyProcessor(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run() } catch (e: RuntimeException) { throw GradleScriptException("Cannot download Kotlin/Native compiler", e) }