dependencies: Implement resuming downloading

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt

@@ -1,16 +1,16 @@
 package org.jetbrains.kotlin.konan.util
 
-import java.io.EOFException
+import java.io.*
-import java.io.File
+import java.net.HttpURLConnection
-import java.io.FileOutputStream
-import java.io.IOException
 import java.net.URL
+import java.net.URLConnection
 import java.nio.file.Files
 import java.nio.file.StandardCopyOption
 
 class DependencyDownloader(
         var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
-        var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS) {
+        var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS
+) {
 
     enum class ReplacingMode {
         /** Redownload the file and replace the existing one. */
@@ -21,34 +21,75 @@ class DependencyDownloader(
         RETURN_EXISTING
     }
 
-    /** Performs an attempt to download a specified file into the specified location */
+    private fun doDownload(originalUrl: URL,
-    fun tryDownload(url: URL, dstFile: File) {
+                           connection: URLConnection,
-        val connection = url.openConnection()
+                           tmpFile: File,
-        val totalBytes = connection.contentLengthLong
+                           currentBytes: Long,
-        var currentBytes = 0L
+                           totalBytes: Long,
-
+                           append: Boolean) {
-        try {
+        @Suppress("NAME_SHADOWING")
-            url.openStream().use { from ->
+        var currentBytes = currentBytes
-                FileOutputStream(dstFile, false).use { to ->
+        connection.getInputStream().use { from ->
-                    val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
+            FileOutputStream(tmpFile, append).use { to ->
-                    var read = from.read(buffer)
+                val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
-                    while (read != -1) {
+                var read = from.read(buffer)
-                        to.write(buffer, 0, read)
+                while (read != -1) {
-                        currentBytes += read
+                    to.write(buffer, 0, read)
-                        updateProgressMsg(url.toString(), currentBytes, totalBytes)
+                    currentBytes += read
-                        read = from.read(buffer)
+                    updateProgressMsg(originalUrl.toString(), currentBytes, totalBytes)
-                    }
+                    read = from.read(buffer)
-                    if (currentBytes != totalBytes) {
+                }
-                        throw EOFException("The stream closed before end of downloading.")
+                if (currentBytes != totalBytes) {
-                    }
+                    throw EOFException("The stream closed before end of downloading.")
                 }
             }
-        } catch (e: Throwable) {
+        }
-            dstFile.delete()
+    }
+
+    private fun tryHttpDownload(originalUrl: URL, connection: HttpURLConnection, tmpFile: File) {
+        @Suppress("NAME_SHADOWING")
+        var connection = connection
+        connection.connect()
+        val totalBytes = connection.contentLengthLong
+        var currentBytes = 0L
+        if (tmpFile.exists()) {
+            currentBytes = tmpFile.length()
+            if (currentBytes < totalBytes) {
+                connection.disconnect()
+                connection = originalUrl.openConnection() as HttpURLConnection
+                connection.setRequestProperty("range", "bytes=$currentBytes-")
+                connection.connect()
+            } else {
+                tmpFile.delete()
+            }
+        }
+
+        doDownload(originalUrl, connection, tmpFile, currentBytes, totalBytes, true)
+    }
+
+    private fun tryOtherDownload(originalUrl: URL, connection: URLConnection, tmpFile: File) {
+        connection.connect()
+        val currentBytes = 0L
+        val totalBytes = connection.contentLengthLong
+
+        try {
+            doDownload(originalUrl, connection, tmpFile, currentBytes, totalBytes, false)
+        } catch(e: Throwable) {
+            tmpFile.delete()
             throw e
         }
     }
 
+    /** Performs an attempt to download a specified file into the specified location */
+    private fun tryDownload(url: URL, tmpFile: File) {
+        var connection = url.openConnection()
+        if (connection is HttpURLConnection) {
+            tryHttpDownload(url, connection, tmpFile)
+        } else {
+            tryOtherDownload(url, connection, tmpFile)
+        }
+    }
+
     /** Downloads a file from [source] url to [destination]. Returns [destination]. */
     fun download(source: URL,
                  destination: File,
@@ -59,12 +100,17 @@ class DependencyDownloader(
             when (replace) {
                 ReplacingMode.RETURN_EXISTING -> return destination
                 ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
-                // TODO: What if dst is a directory?
             }
         }
-        // TODO: resuming must be here
         val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
 
+        check(!tmpFile.isDirectory) {
+            "A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again."
+        }
+        check(!destination.isDirectory) {
+            "The destination file is a directory: ${tmpFile.canonicalPath}. Remove it and try again."
+        }
+
         var attempt = 1
         var waitTime = 0L
         while (true) {
@@ -78,7 +124,7 @@ class DependencyDownloader(
                 attempt++
                 waitTime += attemptIntervalMs
                 println("Cannot download a dependency: $e\n" +
-                        "Wait ${waitTime.toDouble() / 1000} sec and try again (attempt: $attempt/$maxAttempts).")
+                        "Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts).")
                 // TODO: Wait better
                 Thread.sleep(waitTime)
             }
@@ -103,9 +149,11 @@ class DependencyDownloader(
         }
 
     private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) {
-        print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
+        print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
     }
 
+
+
     companion object {
         const val DEFAULT_MAX_ATTEMPTS = 10
         const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt

@@ -14,7 +14,6 @@ class DependencyExtractor {
     }
 
     private fun extractTarGz(tarGz: File, targetDirectory: File) {
-        println("Extract dependency: ${tarGz.canonicalPath} in ${targetDirectory.canonicalPath}")
         val tarProcess = ProcessBuilder().apply {
             command("tar", "-xzf", tarGz.canonicalPath)
             directory(targetDirectory)

shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt

@@ -149,10 +149,10 @@ class DependencyProcessor(dependenciesRoot: File,
                     Cannot find a dependency locally: $dependency.
                     Set `airplaneMode = false` in konan.properties to download it.
                 """.trimIndent())
-            } else {
-                downloader.download(url, archive)
             }
+            downloader.download(url, archive)
         }
+        println("Extracting dependency: $archive into $dependenciesDirectory")
         extractor.extract(archive, dependenciesDirectory)
         extractedDependencies.addWithSave(depName)
     }