[K/N lib] IDEA plugin for K/N + missed items in konan-serializer module

This commit is contained in:
Dmitriy Dolovov
2018-08-27 17:21:04 +03:00
committed by Mikhail Glukhikh
parent 51bb408c56
commit eda29a48a4
44 changed files with 2230 additions and 21 deletions
@@ -0,0 +1,26 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.library
import java.nio.file.Path
import java.nio.file.Paths
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
const val KONAN_STDLIB_NAME = "stdlib"
val KONAN_COMMON_LIBS_PATH: Path
get() = Paths.get("klib", "common")
val KONAN_ALL_PLATFORM_LIBS_PATH: Path
get() = Paths.get("klib", "platform")
fun konanCommonLibraryPath(libraryName: String): Path = KONAN_COMMON_LIBS_PATH.resolve(libraryName)
fun konanSpecificPlatformLibrariesPath(platform: String): Path = KONAN_ALL_PLATFORM_LIBS_PATH.resolve(platform)
fun konanPlatformLibraryPath(platform: String, libraryName: String): Path = konanSpecificPlatformLibrariesPath(platform).resolve(libraryName)
@@ -8,17 +8,12 @@ package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
const val KONAN_STDLIB_NAME = "stdlib"
interface SearchPathResolver {
val searchRoots: List<File>
fun resolve(givenPath: String): File
fun defaultLinks(noStdLib: Boolean, noDefaultLibs: Boolean): List<File>
}
interface SearchPathResolverWithTarget: SearchPathResolver {
interface SearchPathResolverWithTarget : SearchPathResolver {
val target: KonanTarget
}
@@ -0,0 +1,216 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.util
import java.io.*
import java.net.HttpURLConnection
import java.net.URL
import java.net.URLConnection
import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.*
internal typealias ProgressCallback = (url: String, currentBytes: Long, totalBytes: Long) -> Unit
internal class DependencyDownloader(
var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS,
customProgressCallback: ProgressCallback? = null
) {
private val progressCallback = customProgressCallback ?: { url, currentBytes, totalBytes ->
print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
}
val executor = ExecutorCompletionService<Unit>(Executors.newSingleThreadExecutor { r ->
val thread = Thread(r)
thread.name = "konan-dependency-downloader"
thread.isDaemon = true
thread
})
enum class ReplacingMode {
/** Re-download the file and replace the existing one. */
REPLACE,
/** Throw FileAlreadyExistsException */
THROW,
/** Don't download the file and return the existing one*/
RETURN_EXISTING
}
class DownloadingProgress(@Volatile var currentBytes: Long) {
fun update(readBytes: Int) {
currentBytes += readBytes
}
}
private fun HttpURLConnection.checkHTTPResponse(expected: Int, originalUrl: URL = url) {
if (responseCode != expected) {
throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode)
}
}
private fun HttpURLConnection.checkHTTPResponse(originalUrl: URL, predicate: (Int) -> Boolean) {
if (!predicate(responseCode)) {
throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode)
}
}
private fun doDownload(
originalUrl: URL,
connection: URLConnection,
tmpFile: File,
currentBytes: Long,
totalBytes: Long,
append: Boolean
) {
val progress = DownloadingProgress(currentBytes)
// TODO: Implement multi-thread downloading.
executor.submit {
connection.getInputStream().use { from ->
FileOutputStream(tmpFile, append).use { to ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var read = from.read(buffer)
while (read != -1) {
if (Thread.interrupted()) {
throw InterruptedException()
}
to.write(buffer, 0, read)
progress.update(read)
read = from.read(buffer)
}
if (progress.currentBytes != totalBytes) {
throw EOFException("The stream closed before end of downloading.")
}
}
}
}
var result: Future<Unit>?
do {
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
result = executor.poll(1, TimeUnit.SECONDS)
} while (result == null)
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
try {
result.get()
} catch (e: ExecutionException) {
throw e.cause ?: e
}
}
private fun resumeDownload(originalUrl: URL, originalConnection: HttpURLConnection, tmpFile: File) {
originalConnection.connect()
val totalBytes = originalConnection.contentLengthLong
val currentBytes = tmpFile.length()
if (currentBytes >= totalBytes || originalConnection.getHeaderField("Accept-Ranges") != "bytes") {
// The temporary file is bigger then expected or the server doesn't support resuming downloading.
// Download the file from scratch.
doDownload(originalUrl, originalConnection, tmpFile, 0, totalBytes, false)
} else {
originalConnection.disconnect()
val rangeConnection = originalUrl.openConnection() as HttpURLConnection
rangeConnection.setRequestProperty("range", "bytes=$currentBytes-")
rangeConnection.connect()
rangeConnection.checkHTTPResponse(originalUrl) {
it == HttpURLConnection.HTTP_PARTIAL || it == HttpURLConnection.HTTP_OK
}
doDownload(originalUrl, rangeConnection, tmpFile, currentBytes, totalBytes, true)
}
}
/** Performs an attempt to download a specified file into the specified location */
private fun tryDownload(url: URL, tmpFile: File) {
val connection = url.openConnection()
(connection as? HttpURLConnection)?.checkHTTPResponse(HttpURLConnection.HTTP_OK, url)
if (connection is HttpURLConnection && tmpFile.exists()) {
resumeDownload(url, connection, tmpFile)
} else {
connection.connect()
val totalBytes = connection.contentLengthLong
doDownload(url, connection, tmpFile, 0, totalBytes, false)
}
}
/** Downloads a file from [source] url to [destination]. Returns [destination]. */
fun download(
source: URL,
destination: File,
replace: ReplacingMode = ReplacingMode.RETURN_EXISTING
): File {
if (destination.exists()) {
when (replace) {
ReplacingMode.RETURN_EXISTING -> return destination
ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
ReplacingMode.REPLACE -> Unit // Just continue with downloading.
}
}
val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
check(!tmpFile.isDirectory) {
"A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again."
}
check(!destination.isDirectory) {
"The destination file is a directory: ${destination.canonicalPath}. Remove it and try again."
}
var attempt = 1
var waitTime = 0L
while (true) {
try {
tryDownload(source, tmpFile)
break
} catch (e: DependencyDownloaderHTTPResponseException) {
throw e
} catch (e: IOException) {
if (attempt >= maxAttempts) {
throw e
}
attempt++
waitTime += attemptIntervalMs
println(
"Cannot download a dependency: $e\n" +
"Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)."
)
// TODO: Wait better
Thread.sleep(waitTime)
}
}
Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING)
println("Done.")
return destination
}
private val Long.humanReadable: String
get() {
if (this < 0) {
return "-"
}
if (this < 1024) {
return "$this bytes"
}
val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
val prefix = "kMGTPE"[exp - 1]
return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
}
companion object {
const val DEFAULT_MAX_ATTEMPTS = 10
const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L
const val TMP_SUFFIX = "part"
}
}
class DependencyDownloaderHTTPResponseException(val url: URL, val responseCode: Int) :
IOException("Server returned HTTP response code: $responseCode for URL: $url")
@@ -0,0 +1,38 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.unzipTo
import java.io.File
internal class DependencyExtractor {
private val useZip = System.getProperty("os.name").startsWith("Windows")
internal val archiveExtension = if (useZip) {
"zip"
} else {
"tar.gz"
}
private fun extractTarGz(tarGz: File, targetDirectory: File) {
val tarProcess = ProcessBuilder().apply {
command("tar", "-xzf", tarGz.canonicalPath)
directory(targetDirectory)
}.start()
tarProcess.waitFor()
if (tarProcess.exitValue() != 0) {
throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
}
}
fun extract(archive: File, targetDirectory: File) {
if (useZip) {
archive.toPath().unzipTo(targetDirectory.toPath())
} else {
extractTarGz(archive, targetDirectory)
}
}
}
@@ -0,0 +1,284 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.use
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.propertyList
import java.io.File
import java.io.FileNotFoundException
import java.io.RandomAccessFile
import java.net.InetAddress
import java.net.URL
import java.net.UnknownHostException
import java.nio.file.Paths
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
private val Properties.dependenciesUrl: String
get() = getProperty("dependenciesUrl")
?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl")
private val Properties.airplaneMode: Boolean
get() = getProperty("airplaneMode")?.toBoolean() ?: false
private val Properties.downloadingAttempts: Int
get() = getProperty("downloadingAttempts")?.toInt()
?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS
private val Properties.downloadingAttemptIntervalMs: Long
get() = getProperty("downloadingAttemptPauseMs")?.toLong()
?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS
private fun Properties.findCandidates(dependencies: List<String>): Map<String, List<DependencySource>> {
val dependencyProfiles = this.propertyList("dependencyProfiles")
return dependencies.map { dependency ->
dependency to dependencyProfiles.flatMap { profile ->
val candidateSpecs = propertyList("$dependency.$profile")
if (profile == "default" && candidateSpecs.isEmpty()) {
listOf(DependencySource.Remote.Public)
} else {
candidateSpecs.map { candidateSpec ->
when (candidateSpec) {
"remote:public" -> DependencySource.Remote.Public
"remote:internal" -> DependencySource.Remote.Internal
else -> DependencySource.Local(File(candidateSpec))
}
}
}
}
}.toMap()
}
sealed class DependencySource {
data class Local(val path: File) : DependencySource()
sealed class Remote : DependencySource() {
object Public : Remote()
object Internal : Remote()
}
}
/**
* Inspects dependencies and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
* If [airplaneMode] is true will throw a RuntimeException instead of downloading.
*/
class DependencyProcessor(
dependenciesRoot: File,
val dependenciesUrl: String,
dependencyToCandidates: Map<String, List<DependencySource>>,
homeDependencyCache: File = defaultDependencyCacheDir,
val airplaneMode: Boolean = false,
maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS,
customProgressCallback: ProgressCallback? = null,
val keepUnstable: Boolean = true
) {
val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
val cacheDirectory = homeDependencyCache.apply { mkdirs() }
val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
var showInfo = true
private var isInfoShown = false
private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback)
private val extractor = DependencyExtractor()
private val archiveExtension get() = extractor.archiveExtension
constructor(
dependenciesRoot: File,
properties: Properties,
dependencies: List<String>,
dependenciesUrl: String = properties.dependenciesUrl,
keepUnstable: Boolean = true
) : this(
dependenciesRoot,
dependenciesUrl,
dependencyToCandidates = properties.findCandidates(dependencies),
airplaneMode = properties.airplaneMode,
maxAttempts = properties.downloadingAttempts,
attemptIntervalMs = properties.downloadingAttemptIntervalMs,
keepUnstable = keepUnstable
)
class DependencyFile(directory: File, fileName: String) {
val file = File(directory, fileName).apply { createNewFile() }
private val dependencies = file.readLines().toMutableSet()
fun contains(dependency: String) = dependencies.contains(dependency)
fun add(dependency: String) = dependencies.add(dependency)
fun remove(dependency: String) = dependencies.remove(dependency)
fun removeAndSave(dependency: String) {
remove(dependency)
save()
}
fun addAndSave(dependency: String) {
add(dependency)
save()
}
fun save() {
val writer = file.writer()
writer.use {
dependencies.forEach {
writer.write(it)
writer.write("\n")
}
}
}
}
private fun downloadDependency(dependency: String, baseUrl: String) {
val depDir = File(dependenciesDirectory, dependency)
val depName = depDir.name
val fileName = "$depName.$archiveExtension"
val archive = cacheDirectory.resolve(fileName)
val url = URL("$baseUrl/$fileName")
val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
if (extractedDependencies.contains(depName) &&
depDir.exists() &&
depDir.isDirectory &&
depDir.list().isNotEmpty()
) {
if (!keepUnstable && depDir.list().contains(".unstable")) {
// The downloaded version of the dependency is unstable -> redownload it.
depDir.deleteRecursively()
archive.delete()
extractedDependencies.removeAndSave(dependency)
} else {
return
}
}
if (showInfo && !isInfoShown) {
println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
isInfoShown = true
}
if (!archive.exists()) {
if (airplaneMode) {
throw FileNotFoundException(
"""
Cannot find a dependency locally: $dependency.
Set `airplaneMode = false` in konan.properties to download it.
""".trimIndent()
)
}
downloader.download(url, archive)
}
println("Extracting dependency: $archive into $dependenciesDirectory")
extractor.extract(archive, dependenciesDirectory)
extractedDependencies.addAndSave(depName)
}
companion object {
private val lock = ReentrantLock()
val localKonanDir: File by lazy {
File(System.getenv("KONAN_DATA_DIR") ?: (System.getProperty("user.home") + File.separator + ".konan"))
}
@JvmStatic
val defaultDependenciesRoot: File
get() = localKonanDir.resolve("dependencies")
val defaultDependencyCacheDir: File
get() = localKonanDir.resolve("cache")
}
private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) ->
val candidate = candidates.asSequence().mapNotNull { candidate ->
when (candidate) {
is DependencySource.Local -> candidate.takeIf { it.path.exists() }
DependencySource.Remote.Public -> candidate
DependencySource.Remote.Internal -> candidate.takeIf { InternalServer.isAvailable }
}
}.firstOrNull()
candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}")
dependency to candidate
}.toMap()
private fun resolveDependency(dependency: String): File {
val candidate = resolvedDependencies[dependency]
return when (candidate) {
is DependencySource.Local -> candidate.path
is DependencySource.Remote -> File(dependenciesDirectory, dependency)
null -> error("$dependency not declared as dependency")
}
}
fun resolveRelative(relative: String): File {
val path = Paths.get(relative)
if (path.isAbsolute) error("not a relative path: $relative")
val dependency = path.first().toString()
return resolveDependency(dependency).let {
if (path.nameCount > 1) {
it.toPath().resolve(path.subpath(1, path.nameCount)).toFile()
} else {
it
}
}
}
fun run() = lock.withLock {
RandomAccessFile(lockFile, "rw").channel.lock().use {
resolvedDependencies.forEach { (dependency, candidate) ->
val baseUrl = when (candidate) {
is DependencySource.Local -> null
DependencySource.Remote.Public -> dependenciesUrl
DependencySource.Remote.Internal -> InternalServer.url
}
// TODO: consider using different caches for different remotes.
if (baseUrl != null) {
downloadDependency(dependency, baseUrl)
}
}
}
}
}
internal object InternalServer {
private val host = "repo.labs.intellij.net"
val url = "http://$host/kotlin-native"
private val internalDomain = "labs.intellij.net"
val isAvailable by lazy {
val envKey = "KONAN_USE_INTERNAL_SERVER"
val envValue = System.getenv(envKey)
when (envValue) {
"0" -> false
"1" -> true
null -> checkAccessible()
else -> error("unexpected environment: $envKey=$envValue")
}
}
private fun checkAccessible(): Boolean {
if (!InetAddress.getLocalHost().canonicalHostName.endsWith(".$internalDomain")) {
// Fast path:
return false
}
return try {
InetAddress.getByName(host)
true
} catch (e: UnknownHostException) {
false
}
}
}