dependencies: Move dependency downloader into shared project

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -187,7 +187,7 @@ private fun runCmd(command: Array<String>, workDir: File, verbose: Boolean = fal
 
 private fun maybeExecuteHelper(dependenciesRoot: String, properties: Properties, dependencies: List<String>) {
     try {
-        val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin
+        val kClass = Class.forName("org.jetbrains.kotlin.konan.util.Helper0").kotlin
         @Suppress("UNCHECKED_CAST")
         val ctor = kClass.constructors.single() as KFunction<Runnable>
         val result = ctor.call(dependenciesRoot, properties, dependencies)

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -33,28 +33,9 @@ import org.jetbrains.kotlin.config.addKotlinSourceRoots
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.TargetManager
 import org.jetbrains.kotlin.utils.KotlinPaths
+import java.io.File
 import kotlin.reflect.KFunction
 
-// TODO: Don't use reflection?
-private fun maybeExecuteHelper(targetName: String) {
-    try {
-        val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin
-        @Suppress("UNCHECKED_CAST")
-        val ctor = kClass.constructors.single() as KFunction<Runnable>
-        val distribution = Distribution(TargetManager(targetName))
-        val result = ctor.call(
-                distribution.dependenciesDir,
-                distribution.properties,
-                distribution.dependencies
-        )
-        result.run()
-    } catch (notFound: ClassNotFoundException) {
-        // Just ignore, no helper.
-    } catch (e: Throwable) {
-        throw IllegalStateException("Cannot download dependencies.", e)
-    }
-}
-
 class K2Native : CLICompiler<K2NativeCompilerArguments>() {
 
     override fun doExecute(@NotNull arguments: K2NativeCompilerArguments, @NotNull configuration: CompilerConfiguration, @NotNull rootDisposable: Disposable, @Nullable paths: KotlinPaths?): ExitCode {
@@ -85,7 +66,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
         return this?.asList<String>() ?: listOf<String>()
     }
 
-
     // It is executed before doExecute().
     override fun setupPlatformSpecificArgumentsAndServices(
             configuration: CompilerConfiguration,
@@ -152,8 +132,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                 put(ENABLE_ASSERTIONS, arguments.enableAssertions)
             }
         }
-
-        maybeExecuteHelper(arguments.target ?: "host")
     }
 
     override fun createArguments(): K2NativeCompilerArguments {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt

@@ -18,11 +18,11 @@
 // Example startup helper for Kotlin N compiler. Compile to JAR and add it to
 // Kotlin N classpath - it will get loaded and executed automatically.
 //
-package org.jetbrains.kotlin.konan
+package org.jetbrains.kotlin.konan.util
 
 import java.util.*
 
-// Name it CompilerHelper0 so that it get loaded.
+// Name it Helper0 so that it get loaded.
 class Helper0Disabled(val dependenciesDir: String,
                       val properties: Properties,
                       val dependencies: List<String>) : Runnable {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.target.TargetManager.*
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind.*
+import org.jetbrains.kotlin.konan.util.DependencyProcessor
 
 class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
 
@@ -47,9 +48,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
             error("Target $target is not available on the ${TargetManager.host} host")
         }
     }
+
+    private fun Distribution.prepareDependencies() {
+        DependencyProcessor(java.io.File(dependenciesDir), targetProperties).run()
+    }
+
     internal val distribution = Distribution(targetManager, 
         configuration.get(KonanConfigKeys.PROPERTY_FILE),
-        configuration.get(KonanConfigKeys.RUNTIME_FILE))
+        configuration.get(KonanConfigKeys.RUNTIME_FILE)).apply {
+        prepareDependencies()
+    }
 
     private val produce = configuration.get(KonanConfigKeys.PRODUCE)!!
     private val suffix = produce.suffix(targetManager.target)

backend.native/konan.properties

@@ -22,7 +22,8 @@ dependenciesUrl = http://download.jetbrains.com/kotlin/native
 airplaneMode = false
 
 downloadingAttempts = 10
-downloadingAttemptPauseMs = 3000
+downloadingAttemptIntervalMs = 3000
+homeDependencyCache = .konan/cache
 
 llvmDebugOptFlags = -O0
 

build.gradle

@@ -143,7 +143,6 @@ task list_dist(dependsOn: "listDist")
 
 task distCompiler(type: Copy) {
     dependsOn ':backend.native:jars'
-    dependsOn ':tools:helpers:jar'
     dependsOn ':utilities:jar'
     dependsOn ':klib:jar'
     dependsOn ':shared:jar'
@@ -212,11 +211,6 @@ task distCompiler(type: Copy) {
     from(konanPropertiesFile) {
         into('konan')
     }
-
-    // TODO: check if we use native libraries copied to dist (see above) or not.
-    from(project(':tools:helpers').file('build/libs')) {
-        into('konan/lib')
-    }
 }
 
 task listDist(type: Exec) {

buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -100,8 +100,6 @@ abstract class KonanTest extends JavaExec {
         try {
             main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
             classpath = project.configurations.cli_bc
-            // TODO: Remove helpers classpath when downloading is moved into 'shared' project.
-            classpath += project.project(":tools:helpers").sourceSets.main.runtimeClasspath
             enableAssertions = true
             args = ["-output", output,
                     *filesToCompile,

cmd/run_konan

@@ -79,11 +79,10 @@ KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"
 STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar"
 INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar"
 INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar"
-HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar"
 SHARED_JAR="${KONAN_HOME}/konan/lib/shared.jar"
 KLIB_JAR="${KONAN_HOME}/konan/lib/klib.jar"
 UTILITIES_JAR="${KONAN_HOME}/konan/lib/utilities.jar"
-KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$HELPERS_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR"
+KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR"
 TOOL_CLASS=org.jetbrains.kotlin.cli.utilities.MainKt
 
 LIBCLANG_DISABLE_CRASH_RECOVERY=1 \

dependencies/build.gradle

@@ -16,7 +16,12 @@
 
 
 import groovy.transform.stc.ClosureParams
+import groovy.transform.stc.ClosureSignatureHint
+import groovy.transform.stc.FirstParam
+import groovy.transform.stc.FromString
 import org.jetbrains.kotlin.konan.target.*
+import org.jetbrains.kotlin.konan.util.DependencyProcessor
+
 import static org.jetbrains.kotlin.konan.target.KonanTarget.*
 import org.jetbrains.kotlin.konan.properties.KonanProperties
 import org.jetbrains.kotlin.konan.properties.*
@@ -166,23 +171,11 @@ class TgzNativeDep extends NativeDep {
 
 class HelperNativeDep extends TgzNativeDep {
 
-    int maxAttempts = Integer.parseInt(
+    Properties konanProperties = new Properties()
-            project.rootProject.ext.konanProperties['downloadingAttempts'].toString())
-    long attemptPauseMs = Long.parseLong(
-            project.rootProject.ext.konanProperties['downloadingAttemptPauseMs'].toString())
-
-    public HelperNativeDep() {
-        dependsOn(':tools:helpers:jar')
-    }
 
     @TaskAction
     public void downloadAndExtract() {
-        project.javaexec {
+        new DependencyProcessor(baseOutDir, konanProperties, [baseName], baseUrl).run()
-            main = "org.jetbrains.kotlin.konan.MainKt"
-            classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime")
-            classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime").artifacts.files
-            args baseOutDir.canonicalPath, baseUrl, maxAttempts, attemptPauseMs, baseName
-        }
     }
 }
 
@@ -195,8 +188,8 @@ enum DependencyKind {
 
 
     DependencyKind(String name,
-            @ClosureParams(KonanProperties.class) Closure<String> dirGetter,
+            @ClosureParams(value = FromString.class, options = "KonanProperties") Closure<String> dirGetter,
-            @ClosureParams(KonanTarget.class) Closure<String> propertyNameGetter =
+            @ClosureParams(value = FromString.class, options = "KonanTarget") Closure<String> propertyNameGetter =
                            {target -> "${target.userName}${name.capitalize()}Dir"}) {
         this.name = name
         this.dirGetter = dirGetter
@@ -225,8 +218,11 @@ TargetManager.enabled.each { target ->
     )
 
     konanProperties.dependencies.each { dependency ->
-        task "${target.userName}-${dependency}"(type: HelperNativeDep) {
+        if (tasks.findByName(dependency) == null) {
-            baseName = dependency
+            task "${dependency}"(type: HelperNativeDep) {
+                baseName = dependency
+                it.konanProperties = rootProject.ext.konanProperties
+            }
         }
     }
 

settings.gradle

@@ -26,6 +26,5 @@ include ':runtime'
 include ':common'
 include ':backend.native:tests'
 include ':shared'
-include ':tools:helpers'
 include ':tools:kotlin-native-gradle-plugin'
 include ':utilities'

shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt

@@ -182,7 +182,7 @@ fun Path.recursiveCopyTo(destPath: Path) {
         val relative = sourcePath.relativize(oldPath)
         val destFs = destPath.getFileSystem()
         // We are copying files between file systems, 
-        // so pass the relative path through the Sting.
+        // so pass the relative path through the String.
         val newPath = destFs.getPath(destPath.toString(), relative.toString())
 
         // File systems don't allow replacing an existing root.
@@ -195,7 +195,7 @@ fun Path.recursiveCopyTo(destPath: Path) {
 fun bufferedReader(errorStream: InputStream?) = BufferedReader(InputStreamReader(errorStream))
 
 // stdlib `use` function adapted for AutoCloseable.
-private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
+inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
     var closed = false
     try {
         return block(this)
@@ -212,3 +212,11 @@ private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
         }
     }
 }
+
+fun Path.unzipTo(directory: Path) {
+    val zipUri = URI.create("jar:" + this.toUri())
+    FileSystems.newFileSystem(zipUri, emptyMap<String, Any?>(), null).use { zipfs ->
+        val zipPath = zipfs.getPath("/")
+        zipPath.recursiveCopyTo(directory)
+    }
+}

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt

@@ -0,0 +1,115 @@
+package org.jetbrains.kotlin.konan.util
+
+import java.io.EOFException
+import java.io.File
+import java.io.FileOutputStream
+import java.io.IOException
+import java.net.URL
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+
+class DependencyDownloader(
+        var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
+        var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS) {
+
+    enum class ReplacingMode {
+        /** Redownload the file and replace the existing one. */
+        REPLACE,
+        /** Throw FileAlreadyExistsException */
+        THROW,
+        /** Don't download the file and return the existing one*/
+        RETURN_EXISTING
+    }
+
+    /** Performs an attempt to download a specified file into the specified location */
+    fun tryDownload(url: URL, dstFile: File) {
+        val connection = url.openConnection()
+        val totalBytes = connection.contentLengthLong
+        var currentBytes = 0L
+
+        try {
+            url.openStream().use { from ->
+                FileOutputStream(dstFile, false).use { to ->
+                    val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+                    var read = from.read(buffer)
+                    while (read != -1) {
+                        to.write(buffer, 0, read)
+                        currentBytes += read
+                        updateProgressMsg(url.toString(), currentBytes, totalBytes)
+                        read = from.read(buffer)
+                    }
+                    if (currentBytes != totalBytes) {
+                        throw EOFException("The stream closed before end of downloading.")
+                    }
+                }
+            }
+        } catch (e: Throwable) {
+            dstFile.delete()
+            throw e
+        }
+    }
+
+    /** Downloads a file from [source] url to [destination]. Returns [destination]. */
+    fun download(source: URL,
+                 destination: File,
+                 replace: ReplacingMode = ReplacingMode.RETURN_EXISTING): File {
+
+        if (destination.exists()) {
+            @Suppress("NON_EXHAUSTIVE_WHEN")
+            when (replace) {
+                ReplacingMode.RETURN_EXISTING -> return destination
+                ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
+                // TODO: What if dst is a directory?
+            }
+        }
+        // TODO: resuming must be here
+        val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
+
+        var attempt = 1
+        var waitTime = 0L
+        while (true) {
+            try {
+                tryDownload(source, tmpFile)
+                break
+            } catch (e: IOException) {
+                if (attempt >= maxAttempts) {
+                    throw e
+                }
+                attempt++
+                waitTime += attemptIntervalMs
+                println("Cannot download a dependency: $e\n" +
+                        "Wait ${waitTime.toDouble() / 1000} sec and try again (attempt: $attempt/$maxAttempts).")
+                // TODO: Wait better
+                Thread.sleep(waitTime)
+            }
+        }
+
+        Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING)
+        println("Done.")
+        return destination
+    }
+
+    private val Long.humanReadable: String
+        get() {
+            if (this < 0) {
+                return "-"
+            }
+            if (this < 1024) {
+                return "$this bytes"
+            }
+            val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
+            val prefix = "kMGTPE"[exp-1]
+            return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
+        }
+
+    private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) {
+        print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
+    }
+
+    companion object {
+        const val DEFAULT_MAX_ATTEMPTS = 10
+        const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L
+
+        const val TMP_SUFFIX = "part"
+    }
+}

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt

@@ -0,0 +1,35 @@
+package org.jetbrains.kotlin.konan.util
+
+import org.jetbrains.kotlin.konan.file.unzipTo
+import java.io.File
+
+
+class DependencyExtractor {
+    internal val useZip = System.getProperty("os.name").startsWith("Windows")
+
+    internal val archiveExtension = if (useZip) {
+        "zip"
+    } else {
+        "tar.gz"
+    }
+
+    private fun extractTarGz(tarGz: File, targetDirectory: File) {
+        println("Extract dependency: ${tarGz.canonicalPath} in ${targetDirectory.canonicalPath}")
+        val tarProcess = ProcessBuilder().apply {
+            command("tar", "-xzf", tarGz.canonicalPath)
+            directory(targetDirectory)
+        }.start()
+        tarProcess.waitFor()
+        if (tarProcess.exitValue() != 0) {
+            throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
+        }
+    }
+
+    fun extract(archive: File, targetDirectory: File) {
+        if (useZip) {
+            archive.toPath().unzipTo(targetDirectory.toPath())
+        } else {
+            extractTarGz(archive, targetDirectory)
+        }
+    }
+}

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt

@@ -0,0 +1,173 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.konan.util
+
+import org.jetbrains.kotlin.konan.file.use
+import org.jetbrains.kotlin.konan.properties.KonanProperties
+import org.jetbrains.kotlin.konan.properties.Properties
+import java.io.File
+import java.io.FileNotFoundException
+import java.io.RandomAccessFile
+import java.net.URL
+import java.nio.file.Paths
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.withLock
+
+private val Properties.dependenciesUrl : String
+    get() = getProperty("dependenciesUrl")
+            ?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl")
+
+private val Properties.airplaneMode : Boolean
+    get() = getProperty("airplaneMode")?.toBoolean() ?: false
+
+private val Properties.downloadingAttempts : Int
+    get() = getProperty("downloadingAttempts")?.toInt()
+            ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS
+
+private val Properties.downloadingAttemptIntervalMs : Long
+    get() = getProperty("downloadingAttemptPauseMs")?.toLong()
+            ?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS
+
+private val Properties.homeDependencyCache : String
+    get() = getProperty("homeDependencyCache") ?: DependencyProcessor.DEFAULT_HOME_DEPENDENCY_CACHE
+
+
+private val KonanProperties.dependenciesUrl : String            get() = properties.dependenciesUrl
+private val KonanProperties.airplaneMode : Boolean              get() = properties.airplaneMode
+private val KonanProperties.downloadingAttempts : Int           get() = properties.downloadingAttempts
+private val KonanProperties.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs
+private val KonanProperties.homeDependencyCache : String        get() = properties.homeDependencyCache
+
+// TODO: Try to use some dependency management system (Ivy?)
+// TODO: Maybe rename.
+/**
+ * Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
+ * If [airplaneMode] is true will throw a RuntimeException instead of downloading.
+ */
+class DependencyProcessor(dependenciesRoot: File,
+                          val dependenciesUrl: String,
+                          val dependencies: Collection<String>,
+                          homeDependencyCache: String = DEFAULT_HOME_DEPENDENCY_CACHE,
+                          val airplaneMode: Boolean = false,
+                          maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
+                          attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS) {
+
+    val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
+    val cacheDirectory = System.getProperty("user.home")?.let {
+        Paths.get(it).resolve(homeDependencyCache).toFile().apply { mkdirs() }
+    } ?: dependenciesRoot
+
+    val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
+
+    private var isInfoShown = false
+
+    // TOOO: Rename pause -> interval
+    private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs)
+    private val extractor = DependencyExtractor()
+
+    private val archiveExtension get() = extractor.archiveExtension
+
+    constructor(dependenciesRoot: File,
+                properties: KonanProperties,
+                dependenciesUrl: String = properties.dependenciesUrl) : this(
+            dependenciesRoot,
+            properties.properties,
+            properties.dependencies,
+            dependenciesUrl)
+
+    constructor(dependenciesRoot: File,
+                properties: Properties,
+                dependencies: List<String>,
+                dependenciesUrl: String = properties.dependenciesUrl) : this(
+            dependenciesRoot,
+            dependenciesUrl,
+            dependencies,
+            airplaneMode = properties.airplaneMode,
+            maxAttempts = properties.downloadingAttempts,
+            attemptIntervalMs = properties.downloadingAttemptIntervalMs)
+
+
+    class DependencyFile(directory: File, fileName: String) {
+        val file = File(directory, fileName).apply { createNewFile() }
+        private val dependencies = file.readLines().toMutableSet()
+
+        fun contains(dependency: String) = dependencies.contains(dependency)
+        fun add(dependency: String) = dependencies.add(dependency)
+        fun addWithSave(dependency: String) {
+            add(dependency)
+            save()
+        }
+
+        fun save() {
+            val writer = file.writer()
+            writer.use {
+                dependencies.forEach {
+                    writer.write(it)
+                    writer.write("\n")
+                }
+            }
+        }
+    }
+
+    private fun processDependency(dependency: String) {
+        val depDir = File(dependenciesDirectory, dependency)
+        val depName = depDir.name
+
+        val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
+        if (extractedDependencies.contains(depName) &&
+                depDir.exists() &&
+                depDir.isDirectory &&
+                depDir.list().isNotEmpty()) {
+            return
+        }
+
+        if (!isInfoShown) {
+            println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
+            isInfoShown = true
+        }
+        val fileName = "$depName.$archiveExtension"
+        val archive = cacheDirectory.resolve(fileName)
+        val url = URL("$dependenciesUrl/$fileName")
+
+        if (!archive.exists()) {
+            if (airplaneMode) {
+                throw FileNotFoundException("""
+                    Cannot find a dependency locally: $dependency.
+                    Set `airplaneMode = false` in konan.properties to download it.
+                """.trimIndent())
+            } else {
+                downloader.download(url, archive)
+            }
+        }
+        extractor.extract(archive, dependenciesDirectory)
+        extractedDependencies.addWithSave(depName)
+    }
+
+    companion object {
+        val lock = ReentrantLock()
+
+        const val DEFAULT_HOME_DEPENDENCY_CACHE = ".konan/cache"
+    }
+
+    fun run() = lock.withLock {
+        RandomAccessFile(lockFile, "rw").channel.lock().use {
+            dependencies.forEach {
+                processDependency(it)
+            }
+        }
+    }
+}

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/Helper0.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.jetbrains.kotlin.konan
+package org.jetbrains.kotlin.konan.util
 
 import java.io.File
 import java.util.*
@@ -24,15 +24,10 @@ class Helper0(val dependenciesDir: String,
               val dependencies: List<String>): Runnable {
 
     override fun run() {
-        DependencyDownloader(
+        DependencyProcessor(File(dependenciesDir),
-                File(dependenciesDir),
+                properties,
-                properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native"),
                 dependencies,
-                airplaneMode = properties.getProperty("airplaneMode")?.toBoolean() ?: false,
+                properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native")
-                maxAttempts = properties.getProperty("downloadingAttempts")?.toInt()
-                        ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
-                attemptPauseMs = properties.getProperty("downloadingAttemptPauseMs")?.toLong()
-                        ?: DependencyDownloader.DEFAULT_ATTEMPT_PAUSE_MS
         ).run()
     }
 }

tools/helpers/build.gradle

@@ -1,166 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.jetbrains.kotlin.NativeInteropPlugin
-import org.jetbrains.kotlin.RunInteropKonanTest
-import org.jetbrains.kotlin.RunKonanTest
-import org.jetbrains.kotlin.TestFailedException
-
-import java.util.concurrent.atomic.AtomicInteger
-
-buildscript {
-    apply from: "$rootDir/gradle/kotlinGradlePlugin.gradle"
-}
-
-// Copied from backend.native project. =====
-repositories {
-    maven { url kotlinCompilerRepo }
-}
-
-configurations.all {
-    // kotlin-compiler module includes Kotlin runtime bundled;
-    // make Gradle aware of this to avoid multiple Kotlin runtimes in classpath:
-    resolutionStrategy.dependencySubstitution {
-        substitute module('org.jetbrains.kotlin:kotlin-runtime') with module(kotlinCompilerModule)
-        substitute module('org.jetbrains.kotlin:kotlin-stdlib')  with module(kotlinCompilerModule)
-        substitute module('org.jetbrains.kotlin:kotlin-reflect') with module(kotlinCompilerModule)
-    }
-}
-// =========================================
-
-apply plugin: 'kotlin'
-apply plugin: NativeInteropPlugin
-
-configurations {
-    cli_bc.extendsFrom compile
-}
-
-dependencies {
-    compile "org.jetbrains.kotlin:kotlin-runtime:$kotlin_version"
-    // We need this configuration to test the helpers.
-    cli_bc jar.outputs.files
-    cli_bc project(path: ':backend.native', configuration: 'cli_bc')
-}
-
-sourceSets {
-    testOutputLocal {
-        output.dir(rootProject.file("${findProject(":backend.native:tests").testOutputRoot}/tools/helpers"))
-    }
-}
-
-def testDependencies = rootProject.file('dist/dependencies')
-
-void prepareDependenciesDir(File testDependencies) {
-    project.delete testDependencies
-    testDependencies.mkdirs()
-    // Delete all sysroots from the dependencies cache to tests their downloading.
-    project.delete(project.fileTree("${System.getProperty("user.home")}/.konan/cache/") {
-        include "*sysroot*.tar.gz"
-    })
-}
-
-kotlinNativeInterop {
-    sysstat {
-        pkg 'sysstat'
-        headers 'sys/stat.h'
-        flavor 'native'
-    }
-}
-
-project.convention.plugins.remoteExec = new org.jetbrains.kotlin.ExecRemote(project)
-
-task testInteropHelper(type: RunInteropKonanTest) {
-    dependsOn jar
-    dependsOn ':dist'
-
-    dependenciesDir = testDependencies
-    goldValue = "0\n0\n"
-    source = "testData/interop.kt"
-    interop = 'sysstat'
-
-    doFirst {
-        prepareDependenciesDir(testDependencies)
-        project.delete(project.file(buildExePath()).parent)
-    }
-    //Interop gradle plugin doesn't save its work log so we cannot use it to check if dependencies were downloaded or not.
-}
-
-task testCompilerHelper(type: RunKonanTest) {
-    dependsOn jar
-    dependsOn ':dist'
-
-    dependenciesDir = testDependencies
-    source = "testData/main.kt"
-
-    doFirst {
-        prepareDependenciesDir(testDependencies)
-        project.delete(project.file(buildExePath()).parent)
-    }
-
-    doLast {
-        if (!file("${buildExePath()}.compilation.log").text.contains("Extract dependency") ||
-            !file("${buildExePath()}.compilation.log").text.contains("Download dependency")) {
-            throw new TestFailedException("Compilation log contains no Extract/Download messages")
-        }
-    }
-}
-
-// Simple test for parallel compiler execution
-task testParallel(type: DefaultTask) {
-    dependsOn jar
-    dependsOn ':dist'
-
-    for (int i=0; i<4; i++) {
-        task("runParallel$i", type: RunKonanTest) {
-            dependenciesDir = testDependencies
-            source = "testData/main.kt"
-        }
-    }
-
-    doFirst {
-        prepareDependenciesDir(testDependencies)
-    }
-
-    doLast {
-        List<Thread> threads = new ArrayList<>()
-        def downloadCnt = new AtomicInteger(0)
-        def extractCnt = new AtomicInteger(0)
-        tasks.withType(RunKonanTest).matching { it.name.startsWith("runParallel") }.each { task ->
-            threads.add(Thread.start {
-                task.executeTest()
-                // The helper prints messages when it is downloading.
-                // We check that there is only one downloading using compilation logs.
-                if (file("${task.buildExePath()}.compilation.log").text.contains("Extract dependency")) {
-                    extractCnt.incrementAndGet()
-                }
-                if (file("${task.buildExePath()}.compilation.log").text.contains("Download dependency")) {
-                    downloadCnt.incrementAndGet()
-                }
-            })
-        }
-        threads.each { it.join() }
-        if (downloadCnt.intValue() != 1 || extractCnt.intValue() != 1) {
-            throw new TestFailedException("Actual downloads: ${downloadCnt.intValue()}, expected: 1\n" +
-                                          "Actial extractions: ${extractCnt.intValue()}, expected: 1")
-        }
-    }
-}
-
-task testHelpers {
-    dependsOn testInteropHelper
-    dependsOn testCompilerHelper
-    dependsOn testParallel
-}

tools/helpers/src/main/kotlin/DependencyDownloader.kt

@@ -1,243 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.konan
-
-import java.io.*
-import java.net.URL
-import java.nio.file.Files
-import java.nio.file.StandardCopyOption
-import java.util.concurrent.locks.ReentrantLock
-import kotlin.concurrent.thread
-import kotlin.concurrent.withLock
-
-// TODO: Try to use some dependency management system (Ivy?)
-/**
- * Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
- * If [airplaneMode] is true will throw a RuntimeException instead of downloading.
- */
-class DependencyDownloader(dependenciesRoot: File,
-                           val dependenciesUrl: String,
-                           val dependencies: List<String>,
-                           val airplaneMode: Boolean = false,
-                           val maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
-                           val attemptPauseMs: Long = DEFAULT_ATTEMPT_PAUSE_MS) {
-
-    val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
-    val cacheDirectory = System.getProperty("user.home")?.let {
-        File("$it/.konan/cache").apply { mkdirs() }
-    } ?: dependenciesRoot
-
-    val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
-
-    var isInfoShown = false
-
-    class DependencyFile(directory: File, fileName: String) {
-        val file = File(directory, fileName).apply { createNewFile() }
-        private val dependencies = file.readLines().toMutableSet()
-
-        fun contains(dependency: String) = dependencies.contains(dependency)
-        fun add(dependency: String) = dependencies.add(dependency)
-        fun addWithSave(dependency: String) {
-            add(dependency)
-            save()
-        }
-
-        fun save() {
-            val writer = file.writer()
-            writer.use {
-                dependencies.forEach {
-                    writer.write(it)
-                    writer.write("\n")
-                }
-            }
-        }
-    }
-
-    private val useZip = System.getProperty("os.name").startsWith("Windows")
-
-    private val archiveExtension = if (useZip) {
-        "zip"
-    } else {
-        "tar.gz"
-    }
-
-    private fun processDependency(dependency: String) {
-        val depDir = File(dependenciesDirectory, dependency)
-        val depName = depDir.name
-
-        val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
-        if (extractedDependencies.contains(depName) &&
-                depDir.exists() &&
-                depDir.isDirectory &&
-                depDir.list().isNotEmpty()) {
-            return
-        }
-
-        if (!isInfoShown) {
-            println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
-            isInfoShown = true
-        }
-
-        val archive = File(cacheDirectory.canonicalPath, "$depName.$archiveExtension")
-        if (!archive.exists()) {
-            if (!airplaneMode) {
-                var attempt = 0
-                while(true) {
-                    try {
-                        download(depName, archive)
-                        break
-                    } catch (e: IOException) {
-                        if (attempt < maxAttempts) {
-                            attempt++
-                            val pauseTime = attempt * attemptPauseMs
-                            println("Downloading error: ${e.message}.\n" +
-                                    "Waiting ${pauseTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)")
-                            Thread.sleep(pauseTime)
-                        } else { throw e }
-                    }
-                }
-            } else {
-                throw RuntimeException("""
-                    Cannot find a dependency locally: $dependency.
-                    Set `airplaneMode = false` in konan.properties to download it.
-                """.trimIndent())
-            }
-        }
-        extract(archive, dependenciesDirectory)
-        extractedDependencies.addWithSave(depName)
-    }
-
-    private fun extract(archive: File, target: File) {
-        if (useZip) {
-            archive.toPath().unzipTo(target.toPath())
-        } else {
-            extractTarGz(archive, target)
-        }
-    }
-
-    private fun extractTarGz(tarGz: File, target: File) {
-        println("Extract dependency: ${tarGz.canonicalPath} in ${target.canonicalPath}")
-        val tarProcess = ProcessBuilder().apply {
-            command("tar", "-xzf", "${tarGz.canonicalPath}")
-            directory(target)
-        }.start()
-        tarProcess.waitFor()
-        if (tarProcess.exitValue() != 0) {
-            throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
-        }
-    }
-
-    private val Long.humanReadable: String
-        get() {
-            if (this < 0) {
-                return "-"
-            }
-            if (this < 1024) {
-                return "$this bytes"
-            }
-            val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
-            val prefix = "kMGTPE"[exp-1]
-            return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
-        }
-
-    private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) {
-        print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
-    }
-
-    private fun download(dependencyName: String, outputFile: File) {
-        val tmpFile = File("${outputFile.canonicalPath}.part")
-        val url = URL("$dependenciesUrl/$dependencyName.$archiveExtension")
-        val connection = url.openConnection()
-        val totalBytes = connection.contentLengthLong
-
-        var currentBytes = 0L
-        var done = false
-        var downloadError: Throwable? = null
-
-        thread {
-            try {
-                url.openStream().use { from ->
-                    FileOutputStream(tmpFile, false).use { to ->
-                        val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
-                        var read = from.read(buffer)
-                        while (read != -1) {
-                            to.write(buffer, 0, read)
-                            currentBytes += read
-                            read = from.read(buffer)
-                        }
-                        if (currentBytes != totalBytes) {
-                            throw EOFException("The stream closed before end of downloading.")
-                        }
-                    }
-                }
-            } catch (e: Throwable) {
-                downloadError = e
-            }
-            done = true
-        }
-
-        // TODO: Improve console logging
-        while (!done) {
-            Thread.sleep(1000) // We can use condition variable here.
-            updateProgressMsg(url.toString(), currentBytes, totalBytes)
-        }
-        println("Done.")
-        if (downloadError != null) {
-            tmpFile.delete()
-            throw IOException("Cannot download dependency: $url", downloadError)
-        }
-        Files.move(
-                tmpFile.toPath(),
-                outputFile.toPath(),
-                StandardCopyOption.REPLACE_EXISTING
-        )
-    }
-
-    // stdlib `use` function adapted for AutoCloseable.
-    private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
-        var closed = false
-        try {
-            return block(this)
-        } catch (e: Exception) {
-            closed = true
-            try {
-                this?.close()
-            } catch (closeException: Exception) {
-            }
-            throw e
-        } finally {
-            if (!closed) {
-                this?.close()
-            }
-        }
-    }
-
-    companion object {
-        val lock = ReentrantLock()
-
-        const val DEFAULT_MAX_ATTEMPTS = 10
-        const val DEFAULT_ATTEMPT_PAUSE_MS = 3000L
-    }
-
-    fun run() = lock.withLock {
-        RandomAccessFile(lockFile, "rw").channel.lock().use {
-            dependencies.forEach {
-                processDependency(it)
-            }
-        }
-    }
-}

tools/helpers/src/main/kotlin/Unzip.kt

@@ -1,51 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.konan
-
-import java.net.URI
-import java.nio.file.FileSystems
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardCopyOption
-
-fun Path.unzipTo(directory: Path) {
-    val zipUri = URI.create("jar:" + this.toUri())
-    FileSystems.newFileSystem(zipUri, emptyMap<String, Any?>(), null).use { zipfs ->
-        val zipPath = zipfs.getPath("/")
-        zipPath.recursiveCopyTo(directory)
-    }
-}
-
-fun Path.recursiveCopyTo(destPath: Path) {
-    val sourcePath = this
-    Files.walk(sourcePath).forEach { oldPath ->
-
-        val relative = sourcePath.relativize(oldPath)
-
-        // We are copying files between file systems,
-        // so pass the relative path through the Sting.
-        val newPath = destPath.resolve(relative.toString())
-
-        if (Files.isDirectory(oldPath)) {
-            Files.createDirectories(newPath)
-            // TODO: consider copying attributes.
-        } else {
-            Files.copy(oldPath, newPath,
-                    StandardCopyOption.REPLACE_EXISTING)
-        }
-    }
-}

tools/helpers/src/main/kotlin/main.kt

@@ -1,39 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.konan
-
-import java.io.File
-import java.util.*
-
-// TODO: Add command line keys
-// Args: dependencies directory, dependencies url, max attempts, attempt pause, dependencies list
-fun main(args: Array<String>) {
-    if (args.size < 5) {
-        System.exit(1)
-    }
-    val dependenciesDir = File(args[0])
-    val dependenciesUrl = args[1]
-    val maxAttempts = args[2].toInt()
-    val attemptPause = args[3].toLong()
-    val dependencies = List<String>(args.size - 4) { args[4 + it] }
-    DependencyDownloader(dependenciesDir,
-            dependenciesUrl,
-            dependencies,
-            maxAttempts = maxAttempts,
-            attemptPauseMs = attemptPause
-    ).run()
-}

tools/helpers/testData/interop.kt

@@ -1,9 +0,0 @@
-import sysstat.*
-import kotlinx.cinterop.*
-
-fun main(args: Array<String>) {
-    val statBuf = nativeHeap.alloc<statStruct>()
-    val res = stat("/", statBuf.ptr)
-    println(res)
-    println(statBuf.st_uid)
-}

tools/helpers/testData/main.kt

@@ -1,3 +0,0 @@
-fun main(args: Array<String>) {
-    println(42)
-}

tools/kotlin-native-gradle-plugin/build.gradle

@@ -47,11 +47,8 @@ repositories {
     mavenCentral()
 }
 
-evaluationDependsOn(':tools:helpers')
-
 dependencies {
     compile project(':shared')
-    compile project(':tools:helpers')
     compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     compile gradleApi()
 
@@ -69,8 +66,6 @@ test {
 }
 
 jar {
-    dependsOn(':tools:helpers:jar')
-    from (rootProject.findProject(':tools:helpers').sourceSets.main.output)
     from (rootProject.findProject(':shared').sourceSets.main.output)
 }
 

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt

@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.gradle.plugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleScriptException
 import org.gradle.api.tasks.TaskAction
-import org.jetbrains.kotlin.konan.DependencyDownloader
+import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import java.io.File
 
 open class KonanCompilerDownloadTask : DefaultTask() {
@@ -53,7 +53,7 @@ open class KonanCompilerDownloadTask : DefaultTask() {
         try {
             val konanCompiler = project.konanCompilerName()
             logger.info("Downloading Kotlin/Native compiler from $DOWNLOAD_URL/$konanCompiler into $KONAN_PARENT_DIR")
-            DependencyDownloader(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run()
+            DependencyProcessor(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run()
         } catch (e: RuntimeException) {
             throw GradleScriptException("Cannot download Kotlin/Native compiler", e)
         }