azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dependencies: Move dependency downloader into shared project

Browse Source
This commit is contained in:
Ilya Matveev
2017-08-18 17:03:48 +07:00
committed by ilmat192
parent fb15992546
commit 60ed6f6871
23 changed files with 370 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt
+1 -1
View File
@@ -187,7 +187,7 @@ private fun runCmd(command: Array<String>, workDir: File, verbose: Boolean = fal
private fun maybeExecuteHelper(dependenciesRoot: String, properties: Properties, dependencies: List<String>) {
try {
val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin
val kClass = Class.forName("org.jetbrains.kotlin.konan.util.Helper0").kotlin
@Suppress("UNCHECKED_CAST")
val ctor = kClass.constructors.single() as KFunction<Runnable>
val result = ctor.call(dependenciesRoot, properties, dependencies)
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
+1 -23
View File
@@ -33,28 +33,9 @@ import org.jetbrains.kotlin.config.addKotlinSourceRoots
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.TargetManager
import org.jetbrains.kotlin.utils.KotlinPaths
import java.io.File
import kotlin.reflect.KFunction
// TODO: Don't use reflection?
private fun maybeExecuteHelper(targetName: String) {
try {
val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin
@Suppress("UNCHECKED_CAST")
val ctor = kClass.constructors.single() as KFunction<Runnable>
val distribution = Distribution(TargetManager(targetName))
val result = ctor.call(
distribution.dependenciesDir,
distribution.properties,
distribution.dependencies
)
result.run()
} catch (notFound: ClassNotFoundException) {
// Just ignore, no helper.
} catch (e: Throwable) {
throw IllegalStateException("Cannot download dependencies.", e)
}
}
class K2Native : CLICompiler<K2NativeCompilerArguments>() {
override fun doExecute(@NotNull arguments: K2NativeCompilerArguments, @NotNull configuration: CompilerConfiguration, @NotNull rootDisposable: Disposable, @Nullable paths: KotlinPaths?): ExitCode {
@@ -85,7 +66,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
return this?.asList<String>() ?: listOf<String>()
}
// It is executed before doExecute().
override fun setupPlatformSpecificArgumentsAndServices(
configuration: CompilerConfiguration,
@@ -152,8 +132,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
put(ENABLE_ASSERTIONS, arguments.enableAssertions)
}
}
maybeExecuteHelper(arguments.target ?: "host")
}
override fun createArguments(): K2NativeCompilerArguments {
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt
+2 -2
View File
@@ -18,11 +18,11 @@
// Example startup helper for Kotlin N compiler. Compile to JAR and add it to
// Kotlin N classpath - it will get loaded and executed automatically.
//
package org.jetbrains.kotlin.konan
package org.jetbrains.kotlin.konan.util
import java.util.*
// Name it CompilerHelper0 so that it get loaded.
// Name it Helper0 so that it get loaded.
class Helper0Disabled(val dependenciesDir: String,
val properties: Properties,
val dependencies: List<String>) : Runnable {
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
+10 -2
View File
@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.target.TargetManager.*
import org.jetbrains.kotlin.konan.target.CompilerOutputKind.*
import org.jetbrains.kotlin.konan.util.DependencyProcessor
class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
@@ -47,9 +48,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
error("Target $target is not available on the ${TargetManager.host} host")
}
}
private fun Distribution.prepareDependencies() {
DependencyProcessor(java.io.File(dependenciesDir), targetProperties).run()
}
internal val distribution = Distribution(targetManager,
configuration.get(KonanConfigKeys.PROPERTY_FILE),
configuration.get(KonanConfigKeys.RUNTIME_FILE))
configuration.get(KonanConfigKeys.PROPERTY_FILE),
configuration.get(KonanConfigKeys.RUNTIME_FILE)).apply {
prepareDependencies()
}
private val produce = configuration.get(KonanConfigKeys.PRODUCE)!!
private val suffix = produce.suffix(targetManager.target)
backend.native/konan.properties
+2 -1
View File
@@ -22,7 +22,8 @@ dependenciesUrl = http://download.jetbrains.com/kotlin/native
airplaneMode = false
downloadingAttempts = 10
downloadingAttemptPauseMs = 3000
downloadingAttemptIntervalMs = 3000
homeDependencyCache = .konan/cache
llvmDebugOptFlags = -O0
build.gradle
-6
View File
@@ -143,7 +143,6 @@ task list_dist(dependsOn: "listDist")
task distCompiler(type: Copy) {
dependsOn ':backend.native:jars'
dependsOn ':tools:helpers:jar'
dependsOn ':utilities:jar'
dependsOn ':klib:jar'
dependsOn ':shared:jar'
@@ -212,11 +211,6 @@ task distCompiler(type: Copy) {
from(konanPropertiesFile) {
into('konan')
}
// TODO: check if we use native libraries copied to dist (see above) or not.
from(project(':tools:helpers').file('build/libs')) {
into('konan/lib')
}
}
task listDist(type: Exec) {
buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy
-2
View File
@@ -100,8 +100,6 @@ abstract class KonanTest extends JavaExec {
try {
main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
classpath = project.configurations.cli_bc
// TODO: Remove helpers classpath when downloading is moved into 'shared' project.
classpath += project.project(":tools:helpers").sourceSets.main.runtimeClasspath
enableAssertions = true
args = ["-output", output,
*filesToCompile,
cmd/run_konan
+1 -2
View File
@@ -79,11 +79,10 @@ KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"
STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar"
INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar"
INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar"
HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar"
SHARED_JAR="${KONAN_HOME}/konan/lib/shared.jar"
KLIB_JAR="${KONAN_HOME}/konan/lib/klib.jar"
UTILITIES_JAR="${KONAN_HOME}/konan/lib/utilities.jar"
KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$HELPERS_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR"
KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$STUB_GENERATOR_JAR:$INTEROP_INDEXER_JAR:$KONAN_JAR:$KLIB_JAR:$UTILITIES_JAR:$SHARED_JAR"
TOOL_CLASS=org.jetbrains.kotlin.cli.utilities.MainKt
LIBCLANG_DISABLE_CRASH_RECOVERY=1 \
dependencies/build.gradle
Vendored
+14 -18
View File
@@ -16,7 +16,12 @@
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.ClosureSignatureHint
import groovy.transform.stc.FirstParam
import groovy.transform.stc.FromString
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.DependencyProcessor
import static org.jetbrains.kotlin.konan.target.KonanTarget.*
import org.jetbrains.kotlin.konan.properties.KonanProperties
import org.jetbrains.kotlin.konan.properties.*
@@ -166,23 +171,11 @@ class TgzNativeDep extends NativeDep {
class HelperNativeDep extends TgzNativeDep {
int maxAttempts = Integer.parseInt(
project.rootProject.ext.konanProperties['downloadingAttempts'].toString())
long attemptPauseMs = Long.parseLong(
project.rootProject.ext.konanProperties['downloadingAttemptPauseMs'].toString())
public HelperNativeDep() {
dependsOn(':tools:helpers:jar')
}
Properties konanProperties = new Properties()
@TaskAction
public void downloadAndExtract() {
project.javaexec {
main = "org.jetbrains.kotlin.konan.MainKt"
classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime")
classpath += project.findProject(':tools:helpers').getConfigurations().getByName("runtime").artifacts.files
args baseOutDir.canonicalPath, baseUrl, maxAttempts, attemptPauseMs, baseName
}
new DependencyProcessor(baseOutDir, konanProperties, [baseName], baseUrl).run()
}
}
@@ -195,8 +188,8 @@ enum DependencyKind {
DependencyKind(String name,
@ClosureParams(KonanProperties.class) Closure<String> dirGetter,
@ClosureParams(KonanTarget.class) Closure<String> propertyNameGetter =
@ClosureParams(value = FromString.class, options = "KonanProperties") Closure<String> dirGetter,
@ClosureParams(value = FromString.class, options = "KonanTarget") Closure<String> propertyNameGetter =
{target -> "${target.userName}${name.capitalize()}Dir"}) {
this.name = name
this.dirGetter = dirGetter
@@ -225,8 +218,11 @@ TargetManager.enabled.each { target ->
)
konanProperties.dependencies.each { dependency ->
task "${target.userName}-${dependency}"(type: HelperNativeDep) {
baseName = dependency
if (tasks.findByName(dependency) == null) {
task "${dependency}"(type: HelperNativeDep) {
baseName = dependency
it.konanProperties = rootProject.ext.konanProperties
}
}
}
settings.gradle
-1
View File
@@ -26,6 +26,5 @@ include ':runtime'
include ':common'
include ':backend.native:tests'
include ':shared'
include ':tools:helpers'
include ':tools:kotlin-native-gradle-plugin'
include ':utilities'
shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/NoJavaUtil.kt → shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/NoJavaUtil.kt
+10 -2
View File
@@ -182,7 +182,7 @@ fun Path.recursiveCopyTo(destPath: Path) {
val relative = sourcePath.relativize(oldPath)
val destFs = destPath.getFileSystem()
// We are copying files between file systems,
// so pass the relative path through the Sting.
// so pass the relative path through the String.
val newPath = destFs.getPath(destPath.toString(), relative.toString())
// File systems don't allow replacing an existing root.
@@ -195,7 +195,7 @@ fun Path.recursiveCopyTo(destPath: Path) {
fun bufferedReader(errorStream: InputStream?) = BufferedReader(InputStreamReader(errorStream))
// stdlib `use` function adapted for AutoCloseable.
private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
var closed = false
try {
return block(this)
@@ -212,3 +212,11 @@ private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
}
}
}
fun Path.unzipTo(directory: Path) {
val zipUri = URI.create("jar:" + this.toUri())
FileSystems.newFileSystem(zipUri, emptyMap<String, Any?>(), null).use { zipfs ->
val zipPath = zipfs.getPath("/")
zipPath.recursiveCopyTo(directory)
}
}
shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt
+115
View File
@@ -0,0 +1,115 @@
package org.jetbrains.kotlin.konan.util
import java.io.EOFException
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.net.URL
import java.nio.file.Files
import java.nio.file.StandardCopyOption
class DependencyDownloader(
var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS) {
enum class ReplacingMode {
/** Redownload the file and replace the existing one. */
REPLACE,
/** Throw FileAlreadyExistsException */
THROW,
/** Don't download the file and return the existing one*/
RETURN_EXISTING
}
/** Performs an attempt to download a specified file into the specified location */
fun tryDownload(url: URL, dstFile: File) {
val connection = url.openConnection()
val totalBytes = connection.contentLengthLong
var currentBytes = 0L
try {
url.openStream().use { from ->
FileOutputStream(dstFile, false).use { to ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var read = from.read(buffer)
while (read != -1) {
to.write(buffer, 0, read)
currentBytes += read
updateProgressMsg(url.toString(), currentBytes, totalBytes)
read = from.read(buffer)
}
if (currentBytes != totalBytes) {
throw EOFException("The stream closed before end of downloading.")
}
}
}
} catch (e: Throwable) {
dstFile.delete()
throw e
}
}
/** Downloads a file from [source] url to [destination]. Returns [destination]. */
fun download(source: URL,
destination: File,
replace: ReplacingMode = ReplacingMode.RETURN_EXISTING): File {
if (destination.exists()) {
@Suppress("NON_EXHAUSTIVE_WHEN")
when (replace) {
ReplacingMode.RETURN_EXISTING -> return destination
ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
// TODO: What if dst is a directory?
}
}
// TODO: resuming must be here
val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
var attempt = 1
var waitTime = 0L
while (true) {
try {
tryDownload(source, tmpFile)
break
} catch (e: IOException) {
if (attempt >= maxAttempts) {
throw e
}
attempt++
waitTime += attemptIntervalMs
println("Cannot download a dependency: $e\n" +
"Wait ${waitTime.toDouble() / 1000} sec and try again (attempt: $attempt/$maxAttempts).")
// TODO: Wait better
Thread.sleep(waitTime)
}
}
Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING)
println("Done.")
return destination
}
private val Long.humanReadable: String
get() {
if (this < 0) {
return "-"
}
if (this < 1024) {
return "$this bytes"
}
val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
val prefix = "kMGTPE"[exp-1]
return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
}
private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) {
print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
}
companion object {
const val DEFAULT_MAX_ATTEMPTS = 10
const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L
const val TMP_SUFFIX = "part"
}
}
shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt
+35
View File
@@ -0,0 +1,35 @@
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.unzipTo
import java.io.File
class DependencyExtractor {
internal val useZip = System.getProperty("os.name").startsWith("Windows")
internal val archiveExtension = if (useZip) {
"zip"
} else {
"tar.gz"
}
private fun extractTarGz(tarGz: File, targetDirectory: File) {
println("Extract dependency: ${tarGz.canonicalPath} in ${targetDirectory.canonicalPath}")
val tarProcess = ProcessBuilder().apply {
command("tar", "-xzf", tarGz.canonicalPath)
directory(targetDirectory)
}.start()
tarProcess.waitFor()
if (tarProcess.exitValue() != 0) {
throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
}
}
fun extract(archive: File, targetDirectory: File) {
if (useZip) {
archive.toPath().unzipTo(targetDirectory.toPath())
} else {
extractTarGz(archive, targetDirectory)
}
}
}
shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt
+173
View File
@@ -0,0 +1,173 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.use
import org.jetbrains.kotlin.konan.properties.KonanProperties
import org.jetbrains.kotlin.konan.properties.Properties
import java.io.File
import java.io.FileNotFoundException
import java.io.RandomAccessFile
import java.net.URL
import java.nio.file.Paths
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
private val Properties.dependenciesUrl : String
get() = getProperty("dependenciesUrl")
?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl")
private val Properties.airplaneMode : Boolean
get() = getProperty("airplaneMode")?.toBoolean() ?: false
private val Properties.downloadingAttempts : Int
get() = getProperty("downloadingAttempts")?.toInt()
?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS
private val Properties.downloadingAttemptIntervalMs : Long
get() = getProperty("downloadingAttemptPauseMs")?.toLong()
?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS
private val Properties.homeDependencyCache : String
get() = getProperty("homeDependencyCache") ?: DependencyProcessor.DEFAULT_HOME_DEPENDENCY_CACHE
private val KonanProperties.dependenciesUrl : String get() = properties.dependenciesUrl
private val KonanProperties.airplaneMode : Boolean get() = properties.airplaneMode
private val KonanProperties.downloadingAttempts : Int get() = properties.downloadingAttempts
private val KonanProperties.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs
private val KonanProperties.homeDependencyCache : String get() = properties.homeDependencyCache
// TODO: Try to use some dependency management system (Ivy?)
// TODO: Maybe rename.
/**
* Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
* If [airplaneMode] is true will throw a RuntimeException instead of downloading.
*/
class DependencyProcessor(dependenciesRoot: File,
val dependenciesUrl: String,
val dependencies: Collection<String>,
homeDependencyCache: String = DEFAULT_HOME_DEPENDENCY_CACHE,
val airplaneMode: Boolean = false,
maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS) {
val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
val cacheDirectory = System.getProperty("user.home")?.let {
Paths.get(it).resolve(homeDependencyCache).toFile().apply { mkdirs() }
} ?: dependenciesRoot
val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
private var isInfoShown = false
// TOOO: Rename pause -> interval
private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs)
private val extractor = DependencyExtractor()
private val archiveExtension get() = extractor.archiveExtension
constructor(dependenciesRoot: File,
properties: KonanProperties,
dependenciesUrl: String = properties.dependenciesUrl) : this(
dependenciesRoot,
properties.properties,
properties.dependencies,
dependenciesUrl)
constructor(dependenciesRoot: File,
properties: Properties,
dependencies: List<String>,
dependenciesUrl: String = properties.dependenciesUrl) : this(
dependenciesRoot,
dependenciesUrl,
dependencies,
airplaneMode = properties.airplaneMode,
maxAttempts = properties.downloadingAttempts,
attemptIntervalMs = properties.downloadingAttemptIntervalMs)
class DependencyFile(directory: File, fileName: String) {
val file = File(directory, fileName).apply { createNewFile() }
private val dependencies = file.readLines().toMutableSet()
fun contains(dependency: String) = dependencies.contains(dependency)
fun add(dependency: String) = dependencies.add(dependency)
fun addWithSave(dependency: String) {
add(dependency)
save()
}
fun save() {
val writer = file.writer()
writer.use {
dependencies.forEach {
writer.write(it)
writer.write("\n")
}
}
}
}
private fun processDependency(dependency: String) {
val depDir = File(dependenciesDirectory, dependency)
val depName = depDir.name
val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
if (extractedDependencies.contains(depName) &&
depDir.exists() &&
depDir.isDirectory &&
depDir.list().isNotEmpty()) {
return
}
if (!isInfoShown) {
println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
isInfoShown = true
}
val fileName = "$depName.$archiveExtension"
val archive = cacheDirectory.resolve(fileName)
val url = URL("$dependenciesUrl/$fileName")
if (!archive.exists()) {
if (airplaneMode) {
throw FileNotFoundException("""
Cannot find a dependency locally: $dependency.
Set `airplaneMode = false` in konan.properties to download it.
""".trimIndent())
} else {
downloader.download(url, archive)
}
}
extractor.extract(archive, dependenciesDirectory)
extractedDependencies.addWithSave(depName)
}
companion object {
val lock = ReentrantLock()
const val DEFAULT_HOME_DEPENDENCY_CACHE = ".konan/cache"
}
fun run() = lock.withLock {
RandomAccessFile(lockFile, "rw").channel.lock().use {
dependencies.forEach {
processDependency(it)
}
}
}
}
tools/helpers/src/main/kotlin/Helper0.kt → shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/Helper0.kt
+4 -9
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
package org.jetbrains.kotlin.konan.util
import java.io.File
import java.util.*
@@ -24,15 +24,10 @@ class Helper0(val dependenciesDir: String,
val dependencies: List<String>): Runnable {
override fun run() {
DependencyDownloader(
File(dependenciesDir),
properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native"),
DependencyProcessor(File(dependenciesDir),
properties,
dependencies,
airplaneMode = properties.getProperty("airplaneMode")?.toBoolean() ?: false,
maxAttempts = properties.getProperty("downloadingAttempts")?.toInt()
?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
attemptPauseMs = properties.getProperty("downloadingAttemptPauseMs")?.toLong()
?: DependencyDownloader.DEFAULT_ATTEMPT_PAUSE_MS
properties.getProperty("dependenciesUrl", "https://download.jetbrains.com/kotlin/native")
).run()
}
}
tools/helpers/build.gradle
-166
View File
@@ -1,166 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.jetbrains.kotlin.NativeInteropPlugin
import org.jetbrains.kotlin.RunInteropKonanTest
import org.jetbrains.kotlin.RunKonanTest
import org.jetbrains.kotlin.TestFailedException
import java.util.concurrent.atomic.AtomicInteger
buildscript {
apply from: "$rootDir/gradle/kotlinGradlePlugin.gradle"
}
// Copied from backend.native project. =====
repositories {
maven { url kotlinCompilerRepo }
}
configurations.all {
// kotlin-compiler module includes Kotlin runtime bundled;
// make Gradle aware of this to avoid multiple Kotlin runtimes in classpath:
resolutionStrategy.dependencySubstitution {
substitute module('org.jetbrains.kotlin:kotlin-runtime') with module(kotlinCompilerModule)
substitute module('org.jetbrains.kotlin:kotlin-stdlib') with module(kotlinCompilerModule)
substitute module('org.jetbrains.kotlin:kotlin-reflect') with module(kotlinCompilerModule)
}
}
// =========================================
apply plugin: 'kotlin'
apply plugin: NativeInteropPlugin
configurations {
cli_bc.extendsFrom compile
}
dependencies {
compile "org.jetbrains.kotlin:kotlin-runtime:$kotlin_version"
// We need this configuration to test the helpers.
cli_bc jar.outputs.files
cli_bc project(path: ':backend.native', configuration: 'cli_bc')
}
sourceSets {
testOutputLocal {
output.dir(rootProject.file("${findProject(":backend.native:tests").testOutputRoot}/tools/helpers"))
}
}
def testDependencies = rootProject.file('dist/dependencies')
void prepareDependenciesDir(File testDependencies) {
project.delete testDependencies
testDependencies.mkdirs()
// Delete all sysroots from the dependencies cache to tests their downloading.
project.delete(project.fileTree("${System.getProperty("user.home")}/.konan/cache/") {
include "*sysroot*.tar.gz"
})
}
kotlinNativeInterop {
sysstat {
pkg 'sysstat'
headers 'sys/stat.h'
flavor 'native'
}
}
project.convention.plugins.remoteExec = new org.jetbrains.kotlin.ExecRemote(project)
task testInteropHelper(type: RunInteropKonanTest) {
dependsOn jar
dependsOn ':dist'
dependenciesDir = testDependencies
goldValue = "0\n0\n"
source = "testData/interop.kt"
interop = 'sysstat'
doFirst {
prepareDependenciesDir(testDependencies)
project.delete(project.file(buildExePath()).parent)
}
//Interop gradle plugin doesn't save its work log so we cannot use it to check if dependencies were downloaded or not.
}
task testCompilerHelper(type: RunKonanTest) {
dependsOn jar
dependsOn ':dist'
dependenciesDir = testDependencies
source = "testData/main.kt"
doFirst {
prepareDependenciesDir(testDependencies)
project.delete(project.file(buildExePath()).parent)
}
doLast {
if (!file("${buildExePath()}.compilation.log").text.contains("Extract dependency") ||
!file("${buildExePath()}.compilation.log").text.contains("Download dependency")) {
throw new TestFailedException("Compilation log contains no Extract/Download messages")
}
}
}
// Simple test for parallel compiler execution
task testParallel(type: DefaultTask) {
dependsOn jar
dependsOn ':dist'
for (int i=0; i<4; i++) {
task("runParallel$i", type: RunKonanTest) {
dependenciesDir = testDependencies
source = "testData/main.kt"
}
}
doFirst {
prepareDependenciesDir(testDependencies)
}
doLast {
List<Thread> threads = new ArrayList<>()
def downloadCnt = new AtomicInteger(0)
def extractCnt = new AtomicInteger(0)
tasks.withType(RunKonanTest).matching { it.name.startsWith("runParallel") }.each { task ->
threads.add(Thread.start {
task.executeTest()
// The helper prints messages when it is downloading.
// We check that there is only one downloading using compilation logs.
if (file("${task.buildExePath()}.compilation.log").text.contains("Extract dependency")) {
extractCnt.incrementAndGet()
}
if (file("${task.buildExePath()}.compilation.log").text.contains("Download dependency")) {
downloadCnt.incrementAndGet()
}
})
}
threads.each { it.join() }
if (downloadCnt.intValue() != 1 || extractCnt.intValue() != 1) {
throw new TestFailedException("Actual downloads: ${downloadCnt.intValue()}, expected: 1\n" +
"Actial extractions: ${extractCnt.intValue()}, expected: 1")
}
}
}
task testHelpers {
dependsOn testInteropHelper
dependsOn testCompilerHelper
dependsOn testParallel
}
tools/helpers/src/main/kotlin/DependencyDownloader.kt
-243
View File
@@ -1,243 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
import java.io.*
import java.net.URL
import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.thread
import kotlin.concurrent.withLock
// TODO: Try to use some dependency management system (Ivy?)
/**
* Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
* If [airplaneMode] is true will throw a RuntimeException instead of downloading.
*/
class DependencyDownloader(dependenciesRoot: File,
val dependenciesUrl: String,
val dependencies: List<String>,
val airplaneMode: Boolean = false,
val maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
val attemptPauseMs: Long = DEFAULT_ATTEMPT_PAUSE_MS) {
val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
val cacheDirectory = System.getProperty("user.home")?.let {
File("$it/.konan/cache").apply { mkdirs() }
} ?: dependenciesRoot
val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
var isInfoShown = false
class DependencyFile(directory: File, fileName: String) {
val file = File(directory, fileName).apply { createNewFile() }
private val dependencies = file.readLines().toMutableSet()
fun contains(dependency: String) = dependencies.contains(dependency)
fun add(dependency: String) = dependencies.add(dependency)
fun addWithSave(dependency: String) {
add(dependency)
save()
}
fun save() {
val writer = file.writer()
writer.use {
dependencies.forEach {
writer.write(it)
writer.write("\n")
}
}
}
}
private val useZip = System.getProperty("os.name").startsWith("Windows")
private val archiveExtension = if (useZip) {
"zip"
} else {
"tar.gz"
}
private fun processDependency(dependency: String) {
val depDir = File(dependenciesDirectory, dependency)
val depName = depDir.name
val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
if (extractedDependencies.contains(depName) &&
depDir.exists() &&
depDir.isDirectory &&
depDir.list().isNotEmpty()) {
return
}
if (!isInfoShown) {
println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
isInfoShown = true
}
val archive = File(cacheDirectory.canonicalPath, "$depName.$archiveExtension")
if (!archive.exists()) {
if (!airplaneMode) {
var attempt = 0
while(true) {
try {
download(depName, archive)
break
} catch (e: IOException) {
if (attempt < maxAttempts) {
attempt++
val pauseTime = attempt * attemptPauseMs
println("Downloading error: ${e.message}.\n" +
"Waiting ${pauseTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)")
Thread.sleep(pauseTime)
} else { throw e }
}
}
} else {
throw RuntimeException("""
Cannot find a dependency locally: $dependency.
Set `airplaneMode = false` in konan.properties to download it.
""".trimIndent())
}
}
extract(archive, dependenciesDirectory)
extractedDependencies.addWithSave(depName)
}
private fun extract(archive: File, target: File) {
if (useZip) {
archive.toPath().unzipTo(target.toPath())
} else {
extractTarGz(archive, target)
}
}
private fun extractTarGz(tarGz: File, target: File) {
println("Extract dependency: ${tarGz.canonicalPath} in ${target.canonicalPath}")
val tarProcess = ProcessBuilder().apply {
command("tar", "-xzf", "${tarGz.canonicalPath}")
directory(target)
}.start()
tarProcess.waitFor()
if (tarProcess.exitValue() != 0) {
throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
}
}
private val Long.humanReadable: String
get() {
if (this < 0) {
return "-"
}
if (this < 1024) {
return "$this bytes"
}
val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
val prefix = "kMGTPE"[exp-1]
return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
}
private fun updateProgressMsg(url: String, currentBytes: Long, totalBytes: Long) {
print("\rDownload dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
}
private fun download(dependencyName: String, outputFile: File) {
val tmpFile = File("${outputFile.canonicalPath}.part")
val url = URL("$dependenciesUrl/$dependencyName.$archiveExtension")
val connection = url.openConnection()
val totalBytes = connection.contentLengthLong
var currentBytes = 0L
var done = false
var downloadError: Throwable? = null
thread {
try {
url.openStream().use { from ->
FileOutputStream(tmpFile, false).use { to ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var read = from.read(buffer)
while (read != -1) {
to.write(buffer, 0, read)
currentBytes += read
read = from.read(buffer)
}
if (currentBytes != totalBytes) {
throw EOFException("The stream closed before end of downloading.")
}
}
}
} catch (e: Throwable) {
downloadError = e
}
done = true
}
// TODO: Improve console logging
while (!done) {
Thread.sleep(1000) // We can use condition variable here.
updateProgressMsg(url.toString(), currentBytes, totalBytes)
}
println("Done.")
if (downloadError != null) {
tmpFile.delete()
throw IOException("Cannot download dependency: $url", downloadError)
}
Files.move(
tmpFile.toPath(),
outputFile.toPath(),
StandardCopyOption.REPLACE_EXISTING
)
}
// stdlib `use` function adapted for AutoCloseable.
private inline fun <T : AutoCloseable?, R> T.use(block: (T) -> R): R {
var closed = false
try {
return block(this)
} catch (e: Exception) {
closed = true
try {
this?.close()
} catch (closeException: Exception) {
}
throw e
} finally {
if (!closed) {
this?.close()
}
}
}
companion object {
val lock = ReentrantLock()
const val DEFAULT_MAX_ATTEMPTS = 10
const val DEFAULT_ATTEMPT_PAUSE_MS = 3000L
}
fun run() = lock.withLock {
RandomAccessFile(lockFile, "rw").channel.lock().use {
dependencies.forEach {
processDependency(it)
}
}
}
}
tools/helpers/src/main/kotlin/Unzip.kt
-51
View File
@@ -1,51 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
fun Path.unzipTo(directory: Path) {
val zipUri = URI.create("jar:" + this.toUri())
FileSystems.newFileSystem(zipUri, emptyMap<String, Any?>(), null).use { zipfs ->
val zipPath = zipfs.getPath("/")
zipPath.recursiveCopyTo(directory)
}
}
fun Path.recursiveCopyTo(destPath: Path) {
val sourcePath = this
Files.walk(sourcePath).forEach { oldPath ->
val relative = sourcePath.relativize(oldPath)
// We are copying files between file systems,
// so pass the relative path through the Sting.
val newPath = destPath.resolve(relative.toString())
if (Files.isDirectory(oldPath)) {
Files.createDirectories(newPath)
// TODO: consider copying attributes.
} else {
Files.copy(oldPath, newPath,
StandardCopyOption.REPLACE_EXISTING)
}
}
}
tools/helpers/src/main/kotlin/main.kt
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
import java.io.File
import java.util.*
// TODO: Add command line keys
// Args: dependencies directory, dependencies url, max attempts, attempt pause, dependencies list
fun main(args: Array<String>) {
if (args.size < 5) {
System.exit(1)
}
val dependenciesDir = File(args[0])
val dependenciesUrl = args[1]
val maxAttempts = args[2].toInt()
val attemptPause = args[3].toLong()
val dependencies = List<String>(args.size - 4) { args[4 + it] }
DependencyDownloader(dependenciesDir,
dependenciesUrl,
dependencies,
maxAttempts = maxAttempts,
attemptPauseMs = attemptPause
).run()
}
tools/helpers/testData/interop.kt
-9
View File
@@ -1,9 +0,0 @@
import sysstat.*
import kotlinx.cinterop.*
fun main(args: Array<String>) {
val statBuf = nativeHeap.alloc<statStruct>()
val res = stat("/", statBuf.ptr)
println(res)
println(statBuf.st_uid)
}
tools/helpers/testData/main.kt
-3
View File
@@ -1,3 +0,0 @@
fun main(args: Array<String>) {
println(42)
}
tools/kotlin-native-gradle-plugin/build.gradle
-5
View File
@@ -47,11 +47,8 @@ repositories {
mavenCentral()
}
evaluationDependsOn(':tools:helpers')
dependencies {
compile project(':shared')
compile project(':tools:helpers')
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
compile gradleApi()
@@ -69,8 +66,6 @@ test {
}
jar {
dependsOn(':tools:helpers:jar')
from (rootProject.findProject(':tools:helpers').sourceSets.main.output)
from (rootProject.findProject(':shared').sourceSets.main.output)
}
tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KonanCompilerDownloadTask.kt
+2 -2
View File
@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.gradle.plugin
import org.gradle.api.DefaultTask
import org.gradle.api.GradleScriptException
import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.konan.DependencyDownloader
import org.jetbrains.kotlin.konan.util.DependencyProcessor
import java.io.File
open class KonanCompilerDownloadTask : DefaultTask() {
@@ -53,7 +53,7 @@ open class KonanCompilerDownloadTask : DefaultTask() {
try {
val konanCompiler = project.konanCompilerName()
logger.info("Downloading Kotlin/Native compiler from $DOWNLOAD_URL/$konanCompiler into $KONAN_PARENT_DIR")
DependencyDownloader(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run()
DependencyProcessor(File(KONAN_PARENT_DIR), DOWNLOAD_URL, listOf(konanCompiler)).run()
} catch (e: RuntimeException) {
throw GradleScriptException("Cannot download Kotlin/Native compiler", e)
}