K/N: Clean-up unused K/N stuff
This commit is contained in:
Dmitriy Dolovov
committed by
Mikhail Glukhikh
Mikhail Glukhikh
parent
88638ef655
commit
6701dbb220
@@ -1,216 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.konan.util
|
||||
|
||||
import java.io.*
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.net.URLConnection
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.util.concurrent.*
|
||||
|
||||
internal typealias ProgressCallback = (url: String, currentBytes: Long, totalBytes: Long) -> Unit
|
||||
|
||||
internal class DependencyDownloader(
|
||||
var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
|
||||
var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS,
|
||||
customProgressCallback: ProgressCallback? = null
|
||||
) {
|
||||
|
||||
private val progressCallback = customProgressCallback ?: { url, currentBytes, totalBytes ->
|
||||
print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
|
||||
}
|
||||
|
||||
val executor = ExecutorCompletionService<Unit>(Executors.newSingleThreadExecutor { r ->
|
||||
val thread = Thread(r)
|
||||
thread.name = "konan-dependency-downloader"
|
||||
thread.isDaemon = true
|
||||
|
||||
thread
|
||||
})
|
||||
|
||||
enum class ReplacingMode {
|
||||
/** Re-download the file and replace the existing one. */
|
||||
REPLACE,
|
||||
/** Throw FileAlreadyExistsException */
|
||||
THROW,
|
||||
/** Don't download the file and return the existing one*/
|
||||
RETURN_EXISTING
|
||||
}
|
||||
|
||||
class DownloadingProgress(@Volatile var currentBytes: Long) {
|
||||
fun update(readBytes: Int) {
|
||||
currentBytes += readBytes
|
||||
}
|
||||
}
|
||||
|
||||
private fun HttpURLConnection.checkHTTPResponse(expected: Int, originalUrl: URL = url) {
|
||||
if (responseCode != expected) {
|
||||
throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode)
|
||||
}
|
||||
}
|
||||
|
||||
private fun HttpURLConnection.checkHTTPResponse(originalUrl: URL, predicate: (Int) -> Boolean) {
|
||||
if (!predicate(responseCode)) {
|
||||
throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode)
|
||||
}
|
||||
}
|
||||
|
||||
private fun doDownload(
|
||||
originalUrl: URL,
|
||||
connection: URLConnection,
|
||||
tmpFile: File,
|
||||
currentBytes: Long,
|
||||
totalBytes: Long,
|
||||
append: Boolean
|
||||
) {
|
||||
val progress = DownloadingProgress(currentBytes)
|
||||
|
||||
// TODO: Implement multi-thread downloading.
|
||||
executor.submit {
|
||||
connection.getInputStream().use { from ->
|
||||
FileOutputStream(tmpFile, append).use { to ->
|
||||
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
|
||||
var read = from.read(buffer)
|
||||
while (read != -1) {
|
||||
if (Thread.interrupted()) {
|
||||
throw InterruptedException()
|
||||
}
|
||||
to.write(buffer, 0, read)
|
||||
progress.update(read)
|
||||
read = from.read(buffer)
|
||||
}
|
||||
if (progress.currentBytes != totalBytes) {
|
||||
throw EOFException("The stream closed before end of downloading.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var result: Future<Unit>?
|
||||
do {
|
||||
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
|
||||
result = executor.poll(1, TimeUnit.SECONDS)
|
||||
} while (result == null)
|
||||
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
|
||||
|
||||
try {
|
||||
result.get()
|
||||
} catch (e: ExecutionException) {
|
||||
throw e.cause ?: e
|
||||
}
|
||||
}
|
||||
|
||||
private fun resumeDownload(originalUrl: URL, originalConnection: HttpURLConnection, tmpFile: File) {
|
||||
originalConnection.connect()
|
||||
val totalBytes = originalConnection.contentLengthLong
|
||||
val currentBytes = tmpFile.length()
|
||||
if (currentBytes >= totalBytes || originalConnection.getHeaderField("Accept-Ranges") != "bytes") {
|
||||
// The temporary file is bigger then expected or the server doesn't support resuming downloading.
|
||||
// Download the file from scratch.
|
||||
doDownload(originalUrl, originalConnection, tmpFile, 0, totalBytes, false)
|
||||
} else {
|
||||
originalConnection.disconnect()
|
||||
val rangeConnection = originalUrl.openConnection() as HttpURLConnection
|
||||
rangeConnection.setRequestProperty("range", "bytes=$currentBytes-")
|
||||
rangeConnection.connect()
|
||||
rangeConnection.checkHTTPResponse(originalUrl) {
|
||||
it == HttpURLConnection.HTTP_PARTIAL || it == HttpURLConnection.HTTP_OK
|
||||
}
|
||||
doDownload(originalUrl, rangeConnection, tmpFile, currentBytes, totalBytes, true)
|
||||
}
|
||||
}
|
||||
|
||||
/** Performs an attempt to download a specified file into the specified location */
|
||||
private fun tryDownload(url: URL, tmpFile: File) {
|
||||
val connection = url.openConnection()
|
||||
|
||||
(connection as? HttpURLConnection)?.checkHTTPResponse(HttpURLConnection.HTTP_OK, url)
|
||||
|
||||
if (connection is HttpURLConnection && tmpFile.exists()) {
|
||||
resumeDownload(url, connection, tmpFile)
|
||||
} else {
|
||||
connection.connect()
|
||||
val totalBytes = connection.contentLengthLong
|
||||
doDownload(url, connection, tmpFile, 0, totalBytes, false)
|
||||
}
|
||||
}
|
||||
|
||||
/** Downloads a file from [source] url to [destination]. Returns [destination]. */
|
||||
fun download(
|
||||
source: URL,
|
||||
destination: File,
|
||||
replace: ReplacingMode = ReplacingMode.RETURN_EXISTING
|
||||
): File {
|
||||
|
||||
if (destination.exists()) {
|
||||
when (replace) {
|
||||
ReplacingMode.RETURN_EXISTING -> return destination
|
||||
ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
|
||||
ReplacingMode.REPLACE -> Unit // Just continue with downloading.
|
||||
}
|
||||
}
|
||||
val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
|
||||
|
||||
check(!tmpFile.isDirectory) {
|
||||
"A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again."
|
||||
}
|
||||
check(!destination.isDirectory) {
|
||||
"The destination file is a directory: ${destination.canonicalPath}. Remove it and try again."
|
||||
}
|
||||
|
||||
var attempt = 1
|
||||
var waitTime = 0L
|
||||
while (true) {
|
||||
try {
|
||||
tryDownload(source, tmpFile)
|
||||
break
|
||||
} catch (e: DependencyDownloaderHTTPResponseException) {
|
||||
throw e
|
||||
} catch (e: IOException) {
|
||||
if (attempt >= maxAttempts) {
|
||||
throw e
|
||||
}
|
||||
attempt++
|
||||
waitTime += attemptIntervalMs
|
||||
println(
|
||||
"Cannot download a dependency: $e\n" +
|
||||
"Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)."
|
||||
)
|
||||
// TODO: Wait better
|
||||
Thread.sleep(waitTime)
|
||||
}
|
||||
}
|
||||
|
||||
Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING)
|
||||
println("Done.")
|
||||
return destination
|
||||
}
|
||||
|
||||
private val Long.humanReadable: String
|
||||
get() {
|
||||
if (this < 0) {
|
||||
return "-"
|
||||
}
|
||||
if (this < 1024) {
|
||||
return "$this bytes"
|
||||
}
|
||||
val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
|
||||
val prefix = "kMGTPE"[exp - 1]
|
||||
return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val DEFAULT_MAX_ATTEMPTS = 10
|
||||
const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L
|
||||
|
||||
const val TMP_SUFFIX = "part"
|
||||
}
|
||||
}
|
||||
|
||||
class DependencyDownloaderHTTPResponseException(val url: URL, val responseCode: Int) :
|
||||
IOException("Server returned HTTP response code: $responseCode for URL: $url")
|
||||
@@ -1,56 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.konan.util
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.unzipTo
|
||||
import java.io.File
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
internal class DependencyExtractor {
|
||||
private val useZip = System.getProperty("os.name").startsWith("Windows")
|
||||
|
||||
internal val archiveExtension = if (useZip) {
|
||||
"zip"
|
||||
} else {
|
||||
"tar.gz"
|
||||
}
|
||||
|
||||
private fun extractTarGz(tarGz: File, targetDirectory: File) {
|
||||
val tarProcess = ProcessBuilder().apply {
|
||||
command("tar", "-xzf", tarGz.canonicalPath)
|
||||
directory(targetDirectory)
|
||||
inheritIO()
|
||||
}.start()
|
||||
val finished = tarProcess.waitFor(extractionTimeout, extractionTimeoutUntis)
|
||||
when {
|
||||
finished && tarProcess.exitValue() != 0 ->
|
||||
throw RuntimeException(
|
||||
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
|
||||
"Tar exit code: ${tarProcess.exitValue()}."
|
||||
)
|
||||
!finished -> {
|
||||
tarProcess.destroy()
|
||||
throw RuntimeException(
|
||||
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
|
||||
"Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec."
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun extract(archive: File, targetDirectory: File) {
|
||||
if (useZip) {
|
||||
archive.toPath().unzipTo(targetDirectory.toPath())
|
||||
} else {
|
||||
extractTarGz(archive, targetDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
val extractionTimeout = 3600L
|
||||
val extractionTimeoutUntis = TimeUnit.SECONDS
|
||||
}
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.konan.util
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.use
|
||||
import org.jetbrains.kotlin.konan.properties.Properties
|
||||
import org.jetbrains.kotlin.konan.properties.propertyList
|
||||
import java.io.File
|
||||
import java.io.FileNotFoundException
|
||||
import java.io.RandomAccessFile
|
||||
import java.net.InetAddress
|
||||
import java.net.URL
|
||||
import java.net.UnknownHostException
|
||||
import java.nio.file.Paths
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
private val Properties.dependenciesUrl: String
|
||||
get() = getProperty("dependenciesUrl")
|
||||
?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl")
|
||||
|
||||
private val Properties.airplaneMode: Boolean
|
||||
get() = getProperty("airplaneMode")?.toBoolean() ?: false
|
||||
|
||||
private val Properties.downloadingAttempts: Int
|
||||
get() = getProperty("downloadingAttempts")?.toInt()
|
||||
?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS
|
||||
|
||||
private val Properties.downloadingAttemptIntervalMs: Long
|
||||
get() = getProperty("downloadingAttemptPauseMs")?.toLong()
|
||||
?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS
|
||||
|
||||
private fun Properties.findCandidates(dependencies: List<String>): Map<String, List<DependencySource>> {
|
||||
val dependencyProfiles = this.propertyList("dependencyProfiles")
|
||||
return dependencies.map { dependency ->
|
||||
dependency to dependencyProfiles.flatMap { profile ->
|
||||
val candidateSpecs = propertyList("$dependency.$profile")
|
||||
if (profile == "default" && candidateSpecs.isEmpty()) {
|
||||
listOf(DependencySource.Remote.Public)
|
||||
} else {
|
||||
candidateSpecs.map { candidateSpec ->
|
||||
when (candidateSpec) {
|
||||
"remote:public" -> DependencySource.Remote.Public
|
||||
"remote:internal" -> DependencySource.Remote.Internal
|
||||
else -> DependencySource.Local(File(candidateSpec))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}.toMap()
|
||||
}
|
||||
|
||||
sealed class DependencySource {
|
||||
data class Local(val path: File) : DependencySource()
|
||||
|
||||
sealed class Remote : DependencySource() {
|
||||
object Public : Remote()
|
||||
object Internal : Remote()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspects dependencies and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
|
||||
* If [airplaneMode] is true will throw a RuntimeException instead of downloading.
|
||||
*/
|
||||
class DependencyProcessor(
|
||||
dependenciesRoot: File,
|
||||
val dependenciesUrl: String,
|
||||
dependencyToCandidates: Map<String, List<DependencySource>>,
|
||||
homeDependencyCache: File = defaultDependencyCacheDir,
|
||||
val airplaneMode: Boolean = false,
|
||||
maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
|
||||
attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS,
|
||||
customProgressCallback: ProgressCallback? = null,
|
||||
val keepUnstable: Boolean = true
|
||||
) {
|
||||
|
||||
val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
|
||||
val cacheDirectory = homeDependencyCache.apply { mkdirs() }
|
||||
|
||||
val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
|
||||
|
||||
var showInfo = true
|
||||
private var isInfoShown = false
|
||||
|
||||
private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback)
|
||||
private val extractor = DependencyExtractor()
|
||||
|
||||
private val archiveExtension get() = extractor.archiveExtension
|
||||
|
||||
constructor(
|
||||
dependenciesRoot: File,
|
||||
properties: Properties,
|
||||
dependencies: List<String>,
|
||||
dependenciesUrl: String = properties.dependenciesUrl,
|
||||
keepUnstable: Boolean = true
|
||||
) : this(
|
||||
dependenciesRoot,
|
||||
dependenciesUrl,
|
||||
dependencyToCandidates = properties.findCandidates(dependencies),
|
||||
airplaneMode = properties.airplaneMode,
|
||||
maxAttempts = properties.downloadingAttempts,
|
||||
attemptIntervalMs = properties.downloadingAttemptIntervalMs,
|
||||
keepUnstable = keepUnstable
|
||||
)
|
||||
|
||||
class DependencyFile(directory: File, fileName: String) {
|
||||
val file = File(directory, fileName).apply { createNewFile() }
|
||||
private val dependencies = file.readLines().toMutableSet()
|
||||
|
||||
fun contains(dependency: String) = dependencies.contains(dependency)
|
||||
fun add(dependency: String) = dependencies.add(dependency)
|
||||
fun remove(dependency: String) = dependencies.remove(dependency)
|
||||
|
||||
fun removeAndSave(dependency: String) {
|
||||
remove(dependency)
|
||||
save()
|
||||
}
|
||||
|
||||
fun addAndSave(dependency: String) {
|
||||
add(dependency)
|
||||
save()
|
||||
}
|
||||
|
||||
fun save() {
|
||||
val writer = file.writer()
|
||||
writer.use {
|
||||
dependencies.forEach {
|
||||
writer.write(it)
|
||||
writer.write("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun downloadDependency(dependency: String, baseUrl: String) {
|
||||
val depDir = File(dependenciesDirectory, dependency)
|
||||
val depName = depDir.name
|
||||
|
||||
val fileName = "$depName.$archiveExtension"
|
||||
val archive = cacheDirectory.resolve(fileName)
|
||||
val url = URL("$baseUrl/$fileName")
|
||||
|
||||
val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
|
||||
if (extractedDependencies.contains(depName) &&
|
||||
depDir.exists() &&
|
||||
depDir.isDirectory &&
|
||||
depDir.list().isNotEmpty()
|
||||
) {
|
||||
|
||||
if (!keepUnstable && depDir.list().contains(".unstable")) {
|
||||
// The downloaded version of the dependency is unstable -> redownload it.
|
||||
depDir.deleteRecursively()
|
||||
archive.delete()
|
||||
extractedDependencies.removeAndSave(dependency)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (showInfo && !isInfoShown) {
|
||||
println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
|
||||
isInfoShown = true
|
||||
}
|
||||
|
||||
if (!archive.exists()) {
|
||||
if (airplaneMode) {
|
||||
throw FileNotFoundException(
|
||||
"""
|
||||
Cannot find a dependency locally: $dependency.
|
||||
Set `airplaneMode = false` in konan.properties to download it.
|
||||
""".trimIndent()
|
||||
)
|
||||
}
|
||||
downloader.download(url, archive)
|
||||
}
|
||||
println("Extracting dependency: $archive into $dependenciesDirectory")
|
||||
extractor.extract(archive, dependenciesDirectory)
|
||||
extractedDependencies.addAndSave(depName)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val lock = ReentrantLock()
|
||||
|
||||
val localKonanDir: File by lazy {
|
||||
File(System.getenv("KONAN_DATA_DIR") ?: (System.getProperty("user.home") + File.separator + ".konan"))
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
val defaultDependenciesRoot: File
|
||||
get() = localKonanDir.resolve("dependencies")
|
||||
|
||||
val defaultDependencyCacheDir: File
|
||||
get() = localKonanDir.resolve("cache")
|
||||
}
|
||||
|
||||
private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) ->
|
||||
val candidate = candidates.asSequence().mapNotNull { candidate ->
|
||||
when (candidate) {
|
||||
is DependencySource.Local -> candidate.takeIf { it.path.exists() }
|
||||
DependencySource.Remote.Public -> candidate
|
||||
DependencySource.Remote.Internal -> candidate.takeIf { InternalServer.isAvailable }
|
||||
}
|
||||
}.firstOrNull()
|
||||
|
||||
candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}")
|
||||
|
||||
dependency to candidate
|
||||
}.toMap()
|
||||
|
||||
private fun resolveDependency(dependency: String): File {
|
||||
val candidate = resolvedDependencies[dependency]
|
||||
return when (candidate) {
|
||||
is DependencySource.Local -> candidate.path
|
||||
is DependencySource.Remote -> File(dependenciesDirectory, dependency)
|
||||
null -> error("$dependency not declared as dependency")
|
||||
}
|
||||
}
|
||||
|
||||
fun resolveRelative(relative: String): File {
|
||||
val path = Paths.get(relative)
|
||||
if (path.isAbsolute) error("not a relative path: $relative")
|
||||
|
||||
val dependency = path.first().toString()
|
||||
return resolveDependency(dependency).let {
|
||||
if (path.nameCount > 1) {
|
||||
it.toPath().resolve(path.subpath(1, path.nameCount)).toFile()
|
||||
} else {
|
||||
it
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun run() = lock.withLock {
|
||||
RandomAccessFile(lockFile, "rw").channel.lock().use {
|
||||
resolvedDependencies.forEach { (dependency, candidate) ->
|
||||
val baseUrl = when (candidate) {
|
||||
is DependencySource.Local -> null
|
||||
DependencySource.Remote.Public -> dependenciesUrl
|
||||
DependencySource.Remote.Internal -> InternalServer.url
|
||||
}
|
||||
// TODO: consider using different caches for different remotes.
|
||||
if (baseUrl != null) {
|
||||
downloadDependency(dependency, baseUrl)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal object InternalServer {
|
||||
private val host = "repo.labs.intellij.net"
|
||||
val url = "http://$host/kotlin-native"
|
||||
|
||||
private val internalDomain = "labs.intellij.net"
|
||||
|
||||
val isAvailable by lazy {
|
||||
val envKey = "KONAN_USE_INTERNAL_SERVER"
|
||||
val envValue = System.getenv(envKey)
|
||||
when (envValue) {
|
||||
"0" -> false
|
||||
"1" -> true
|
||||
null -> checkAccessible()
|
||||
else -> error("unexpected environment: $envKey=$envValue")
|
||||
}
|
||||
}
|
||||
|
||||
private fun checkAccessible(): Boolean {
|
||||
if (!InetAddress.getLocalHost().canonicalHostName.endsWith(".$internalDomain")) {
|
||||
// Fast path:
|
||||
return false
|
||||
}
|
||||
|
||||
return try {
|
||||
InetAddress.getByName(host)
|
||||
true
|
||||
} catch (e: UnknownHostException) {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user