From 6701dbb22051f48c2408842c50d06cc3c029990e Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Fri, 7 Sep 2018 12:45:47 +0300 Subject: [PATCH] K/N: Clean-up unused K/N stuff --- .../kotlin/konan/util/DependencyDownloader.kt | 216 ------------- .../kotlin/konan/util/DependencyExtractor.kt | 56 ---- .../kotlin/konan/util/DependencyProcessor.kt | 284 ------------------ 3 files changed, 556 deletions(-) delete mode 100644 konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt delete mode 100644 konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt delete mode 100644 konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt diff --git a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt b/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt deleted file mode 100644 index 60a3b6b3312..00000000000 --- a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.util - -import java.io.* -import java.net.HttpURLConnection -import java.net.URL -import java.net.URLConnection -import java.nio.file.Files -import java.nio.file.StandardCopyOption -import java.util.concurrent.* - -internal typealias ProgressCallback = (url: String, currentBytes: Long, totalBytes: Long) -> Unit - -internal class DependencyDownloader( - var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS, - var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS, - customProgressCallback: ProgressCallback? = null -) { - - private val progressCallback = customProgressCallback ?: { url, currentBytes, totalBytes -> - print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ") - } - - val executor = ExecutorCompletionService(Executors.newSingleThreadExecutor { r -> - val thread = Thread(r) - thread.name = "konan-dependency-downloader" - thread.isDaemon = true - - thread - }) - - enum class ReplacingMode { - /** Re-download the file and replace the existing one. */ - REPLACE, - /** Throw FileAlreadyExistsException */ - THROW, - /** Don't download the file and return the existing one*/ - RETURN_EXISTING - } - - class DownloadingProgress(@Volatile var currentBytes: Long) { - fun update(readBytes: Int) { - currentBytes += readBytes - } - } - - private fun HttpURLConnection.checkHTTPResponse(expected: Int, originalUrl: URL = url) { - if (responseCode != expected) { - throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode) - } - } - - private fun HttpURLConnection.checkHTTPResponse(originalUrl: URL, predicate: (Int) -> Boolean) { - if (!predicate(responseCode)) { - throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode) - } - } - - private fun doDownload( - originalUrl: URL, - connection: URLConnection, - tmpFile: File, - currentBytes: Long, - totalBytes: Long, - append: Boolean - ) { - val progress = DownloadingProgress(currentBytes) - - // TODO: Implement multi-thread downloading. - executor.submit { - connection.getInputStream().use { from -> - FileOutputStream(tmpFile, append).use { to -> - val buffer = ByteArray(DEFAULT_BUFFER_SIZE) - var read = from.read(buffer) - while (read != -1) { - if (Thread.interrupted()) { - throw InterruptedException() - } - to.write(buffer, 0, read) - progress.update(read) - read = from.read(buffer) - } - if (progress.currentBytes != totalBytes) { - throw EOFException("The stream closed before end of downloading.") - } - } - } - } - - var result: Future? - do { - progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes) - result = executor.poll(1, TimeUnit.SECONDS) - } while (result == null) - progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes) - - try { - result.get() - } catch (e: ExecutionException) { - throw e.cause ?: e - } - } - - private fun resumeDownload(originalUrl: URL, originalConnection: HttpURLConnection, tmpFile: File) { - originalConnection.connect() - val totalBytes = originalConnection.contentLengthLong - val currentBytes = tmpFile.length() - if (currentBytes >= totalBytes || originalConnection.getHeaderField("Accept-Ranges") != "bytes") { - // The temporary file is bigger then expected or the server doesn't support resuming downloading. - // Download the file from scratch. - doDownload(originalUrl, originalConnection, tmpFile, 0, totalBytes, false) - } else { - originalConnection.disconnect() - val rangeConnection = originalUrl.openConnection() as HttpURLConnection - rangeConnection.setRequestProperty("range", "bytes=$currentBytes-") - rangeConnection.connect() - rangeConnection.checkHTTPResponse(originalUrl) { - it == HttpURLConnection.HTTP_PARTIAL || it == HttpURLConnection.HTTP_OK - } - doDownload(originalUrl, rangeConnection, tmpFile, currentBytes, totalBytes, true) - } - } - - /** Performs an attempt to download a specified file into the specified location */ - private fun tryDownload(url: URL, tmpFile: File) { - val connection = url.openConnection() - - (connection as? HttpURLConnection)?.checkHTTPResponse(HttpURLConnection.HTTP_OK, url) - - if (connection is HttpURLConnection && tmpFile.exists()) { - resumeDownload(url, connection, tmpFile) - } else { - connection.connect() - val totalBytes = connection.contentLengthLong - doDownload(url, connection, tmpFile, 0, totalBytes, false) - } - } - - /** Downloads a file from [source] url to [destination]. Returns [destination]. */ - fun download( - source: URL, - destination: File, - replace: ReplacingMode = ReplacingMode.RETURN_EXISTING - ): File { - - if (destination.exists()) { - when (replace) { - ReplacingMode.RETURN_EXISTING -> return destination - ReplacingMode.THROW -> throw FileAlreadyExistsException(destination) - ReplacingMode.REPLACE -> Unit // Just continue with downloading. - } - } - val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX") - - check(!tmpFile.isDirectory) { - "A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again." - } - check(!destination.isDirectory) { - "The destination file is a directory: ${destination.canonicalPath}. Remove it and try again." - } - - var attempt = 1 - var waitTime = 0L - while (true) { - try { - tryDownload(source, tmpFile) - break - } catch (e: DependencyDownloaderHTTPResponseException) { - throw e - } catch (e: IOException) { - if (attempt >= maxAttempts) { - throw e - } - attempt++ - waitTime += attemptIntervalMs - println( - "Cannot download a dependency: $e\n" + - "Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)." - ) - // TODO: Wait better - Thread.sleep(waitTime) - } - } - - Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING) - println("Done.") - return destination - } - - private val Long.humanReadable: String - get() { - if (this < 0) { - return "-" - } - if (this < 1024) { - return "$this bytes" - } - val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt() - val prefix = "kMGTPE"[exp - 1] - return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix) - } - - companion object { - const val DEFAULT_MAX_ATTEMPTS = 10 - const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L - - const val TMP_SUFFIX = "part" - } -} - -class DependencyDownloaderHTTPResponseException(val url: URL, val responseCode: Int) : - IOException("Server returned HTTP response code: $responseCode for URL: $url") diff --git a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt b/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt deleted file mode 100644 index 617cd1a5fd5..00000000000 --- a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.util - -import org.jetbrains.kotlin.konan.file.unzipTo -import java.io.File -import java.util.concurrent.TimeUnit - -internal class DependencyExtractor { - private val useZip = System.getProperty("os.name").startsWith("Windows") - - internal val archiveExtension = if (useZip) { - "zip" - } else { - "tar.gz" - } - - private fun extractTarGz(tarGz: File, targetDirectory: File) { - val tarProcess = ProcessBuilder().apply { - command("tar", "-xzf", tarGz.canonicalPath) - directory(targetDirectory) - inheritIO() - }.start() - val finished = tarProcess.waitFor(extractionTimeout, extractionTimeoutUntis) - when { - finished && tarProcess.exitValue() != 0 -> - throw RuntimeException( - "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + - "Tar exit code: ${tarProcess.exitValue()}." - ) - !finished -> { - tarProcess.destroy() - throw RuntimeException( - "Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" + - "Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec." - ) - } - } - } - - fun extract(archive: File, targetDirectory: File) { - if (useZip) { - archive.toPath().unzipTo(targetDirectory.toPath()) - } else { - extractTarGz(archive, targetDirectory) - } - } - - companion object { - val extractionTimeout = 3600L - val extractionTimeoutUntis = TimeUnit.SECONDS - } -} \ No newline at end of file diff --git a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt deleted file mode 100644 index 68d6e79bb54..00000000000 --- a/konan/serializer/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.util - -import org.jetbrains.kotlin.konan.file.use -import org.jetbrains.kotlin.konan.properties.Properties -import org.jetbrains.kotlin.konan.properties.propertyList -import java.io.File -import java.io.FileNotFoundException -import java.io.RandomAccessFile -import java.net.InetAddress -import java.net.URL -import java.net.UnknownHostException -import java.nio.file.Paths -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.withLock - -private val Properties.dependenciesUrl: String - get() = getProperty("dependenciesUrl") - ?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl") - -private val Properties.airplaneMode: Boolean - get() = getProperty("airplaneMode")?.toBoolean() ?: false - -private val Properties.downloadingAttempts: Int - get() = getProperty("downloadingAttempts")?.toInt() - ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS - -private val Properties.downloadingAttemptIntervalMs: Long - get() = getProperty("downloadingAttemptPauseMs")?.toLong() - ?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS - -private fun Properties.findCandidates(dependencies: List): Map> { - val dependencyProfiles = this.propertyList("dependencyProfiles") - return dependencies.map { dependency -> - dependency to dependencyProfiles.flatMap { profile -> - val candidateSpecs = propertyList("$dependency.$profile") - if (profile == "default" && candidateSpecs.isEmpty()) { - listOf(DependencySource.Remote.Public) - } else { - candidateSpecs.map { candidateSpec -> - when (candidateSpec) { - "remote:public" -> DependencySource.Remote.Public - "remote:internal" -> DependencySource.Remote.Internal - else -> DependencySource.Local(File(candidateSpec)) - } - } - } - } - }.toMap() -} - -sealed class DependencySource { - data class Local(val path: File) : DependencySource() - - sealed class Remote : DependencySource() { - object Public : Remote() - object Internal : Remote() - } -} - -/** - * Inspects dependencies and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl]. - * If [airplaneMode] is true will throw a RuntimeException instead of downloading. - */ -class DependencyProcessor( - dependenciesRoot: File, - val dependenciesUrl: String, - dependencyToCandidates: Map>, - homeDependencyCache: File = defaultDependencyCacheDir, - val airplaneMode: Boolean = false, - maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS, - attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS, - customProgressCallback: ProgressCallback? = null, - val keepUnstable: Boolean = true -) { - - val dependenciesDirectory = dependenciesRoot.apply { mkdirs() } - val cacheDirectory = homeDependencyCache.apply { mkdirs() } - - val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() } - - var showInfo = true - private var isInfoShown = false - - private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback) - private val extractor = DependencyExtractor() - - private val archiveExtension get() = extractor.archiveExtension - - constructor( - dependenciesRoot: File, - properties: Properties, - dependencies: List, - dependenciesUrl: String = properties.dependenciesUrl, - keepUnstable: Boolean = true - ) : this( - dependenciesRoot, - dependenciesUrl, - dependencyToCandidates = properties.findCandidates(dependencies), - airplaneMode = properties.airplaneMode, - maxAttempts = properties.downloadingAttempts, - attemptIntervalMs = properties.downloadingAttemptIntervalMs, - keepUnstable = keepUnstable - ) - - class DependencyFile(directory: File, fileName: String) { - val file = File(directory, fileName).apply { createNewFile() } - private val dependencies = file.readLines().toMutableSet() - - fun contains(dependency: String) = dependencies.contains(dependency) - fun add(dependency: String) = dependencies.add(dependency) - fun remove(dependency: String) = dependencies.remove(dependency) - - fun removeAndSave(dependency: String) { - remove(dependency) - save() - } - - fun addAndSave(dependency: String) { - add(dependency) - save() - } - - fun save() { - val writer = file.writer() - writer.use { - dependencies.forEach { - writer.write(it) - writer.write("\n") - } - } - } - } - - private fun downloadDependency(dependency: String, baseUrl: String) { - val depDir = File(dependenciesDirectory, dependency) - val depName = depDir.name - - val fileName = "$depName.$archiveExtension" - val archive = cacheDirectory.resolve(fileName) - val url = URL("$baseUrl/$fileName") - - val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted") - if (extractedDependencies.contains(depName) && - depDir.exists() && - depDir.isDirectory && - depDir.list().isNotEmpty() - ) { - - if (!keepUnstable && depDir.list().contains(".unstable")) { - // The downloaded version of the dependency is unstable -> redownload it. - depDir.deleteRecursively() - archive.delete() - extractedDependencies.removeAndSave(dependency) - } else { - return - } - } - - if (showInfo && !isInfoShown) { - println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.") - isInfoShown = true - } - - if (!archive.exists()) { - if (airplaneMode) { - throw FileNotFoundException( - """ - Cannot find a dependency locally: $dependency. - Set `airplaneMode = false` in konan.properties to download it. - """.trimIndent() - ) - } - downloader.download(url, archive) - } - println("Extracting dependency: $archive into $dependenciesDirectory") - extractor.extract(archive, dependenciesDirectory) - extractedDependencies.addAndSave(depName) - } - - companion object { - private val lock = ReentrantLock() - - val localKonanDir: File by lazy { - File(System.getenv("KONAN_DATA_DIR") ?: (System.getProperty("user.home") + File.separator + ".konan")) - } - - @JvmStatic - val defaultDependenciesRoot: File - get() = localKonanDir.resolve("dependencies") - - val defaultDependencyCacheDir: File - get() = localKonanDir.resolve("cache") - } - - private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) -> - val candidate = candidates.asSequence().mapNotNull { candidate -> - when (candidate) { - is DependencySource.Local -> candidate.takeIf { it.path.exists() } - DependencySource.Remote.Public -> candidate - DependencySource.Remote.Internal -> candidate.takeIf { InternalServer.isAvailable } - } - }.firstOrNull() - - candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}") - - dependency to candidate - }.toMap() - - private fun resolveDependency(dependency: String): File { - val candidate = resolvedDependencies[dependency] - return when (candidate) { - is DependencySource.Local -> candidate.path - is DependencySource.Remote -> File(dependenciesDirectory, dependency) - null -> error("$dependency not declared as dependency") - } - } - - fun resolveRelative(relative: String): File { - val path = Paths.get(relative) - if (path.isAbsolute) error("not a relative path: $relative") - - val dependency = path.first().toString() - return resolveDependency(dependency).let { - if (path.nameCount > 1) { - it.toPath().resolve(path.subpath(1, path.nameCount)).toFile() - } else { - it - } - } - } - - fun run() = lock.withLock { - RandomAccessFile(lockFile, "rw").channel.lock().use { - resolvedDependencies.forEach { (dependency, candidate) -> - val baseUrl = when (candidate) { - is DependencySource.Local -> null - DependencySource.Remote.Public -> dependenciesUrl - DependencySource.Remote.Internal -> InternalServer.url - } - // TODO: consider using different caches for different remotes. - if (baseUrl != null) { - downloadDependency(dependency, baseUrl) - } - } - } - } -} - -internal object InternalServer { - private val host = "repo.labs.intellij.net" - val url = "http://$host/kotlin-native" - - private val internalDomain = "labs.intellij.net" - - val isAvailable by lazy { - val envKey = "KONAN_USE_INTERNAL_SERVER" - val envValue = System.getenv(envKey) - when (envValue) { - "0" -> false - "1" -> true - null -> checkAccessible() - else -> error("unexpected environment: $envKey=$envValue") - } - } - - private fun checkAccessible(): Boolean { - if (!InetAddress.getLocalHost().canonicalHostName.endsWith(".$internalDomain")) { - // Fast path: - return false - } - - return try { - InetAddress.getByName(host) - true - } catch (e: UnknownHostException) { - false - } - } -}