Move everything under kotlin-native folder

I was forced to manually do update the following files, because otherwise
they would be ignored according .gitignore settings. Probably they
should be deleted from repo.

Interop/.idea/compiler.xml
Interop/.idea/gradle.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_runtime_1_0_3.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_stdlib_1_0_3.xml
Interop/.idea/modules.xml
Interop/.idea/modules/Indexer/Indexer.iml
Interop/.idea/modules/Runtime/Runtime.iml
Interop/.idea/modules/StubGenerator/StubGenerator.iml
backend.native/backend.native.iml
backend.native/bc.frontend/bc.frontend.iml
backend.native/cli.bc/cli.bc.iml
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt
backend.native/tests/link/lib/foo.kt
backend.native/tests/link/lib/foo2.kt
backend.native/tests/teamcity-test.property
This commit is contained in:
Stanislav Erokhin
2020-10-27 21:00:28 +03:00
parent 91e4162dad
commit f624800b84
2830 changed files with 0 additions and 0 deletions
+79
View File
@@ -0,0 +1,79 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("UnstableApiUsage")
import org.jetbrains.kotlin.VersionGenerator
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
plugins {
id("kotlin")
}
val rootBuildDirectory by extra(file(".."))
apply(from="../gradle/loadRootProperties.gradle")
val kotlinVersion: String by extra
val buildKotlinVersion: String by extra
val konanVersion: String by extra
val kotlinCompilerRepo: String by extra
val buildKotlinCompilerRepo: String by extra
group = "org.jetbrains.kotlin"
version = konanVersion
repositories {
maven("https://cache-redirector.jetbrains.com/maven-central")
mavenCentral()
maven(kotlinCompilerRepo)
maven(buildKotlinCompilerRepo)
}
// FIXME(ddol): KLIB-REFACTORING-CLEANUP: drop generation of KonanVersion!
val generateCompilerVersion by tasks.registering(VersionGenerator::class) {}
sourceSets["main"].withConvention(KotlinSourceSet::class) {
kotlin.srcDir("src/main/kotlin")
kotlin.srcDir("src/library/kotlin")
kotlin.srcDir(generateCompilerVersion.get().versionSourceDirectory)
}
tasks.withType<KotlinCompile> {
dependsOn(generateCompilerVersion)
kotlinOptions.jvmTarget = "1.8"
}
tasks.clean {
doFirst {
val versionSourceDirectory = generateCompilerVersion.get().versionSourceDirectory
if (versionSourceDirectory.exists()) {
versionSourceDirectory.delete()
}
}
}
tasks.jar {
archiveFileName.set("shared.jar")
}
dependencies {
kotlinCompilerClasspath("org.jetbrains.kotlin:kotlin-compiler-embeddable:$buildKotlinVersion")
implementation("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion")
implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
api("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion")
api("org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion")
}
@@ -0,0 +1,28 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
buildscript {
ext.rootBuildDirectory = "$rootDir/../.."
}
apply plugin: 'java'
repositories {
mavenCentral()
}
dependencies {
compile gradleApi()
}
@@ -0,0 +1,96 @@
package org.jetbrains.kotlin;
import groovy.lang.Closure;
import org.gradle.api.DefaultTask;
import org.gradle.api.Task;
import org.gradle.api.tasks.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class VersionGenerator extends DefaultTask {
@OutputDirectory
public File getVersionSourceDirectory() {
return getProject().file("build/generated");
}
@OutputFile
public File getVersionFile() {
return getProject().file(getVersionSourceDirectory().getPath() + "/org/jetbrains/kotlin/konan/CompilerVersionGenerated.kt");
}
@Input
public String getKonanVersion() {
return getProject().getProperties().get("konanVersion").toString();
}
// TeamCity passes all configuration parameters into a build script as project properties.
// Thus we can use them here instead of environment variables.
@Optional
@Input
public String getBuildNumber() {
Object property = getProject().findProperty("build.number");
if (property == null) return null;
return property.toString();
}
@Input
public String getMeta() {
Object konanMetaVersionProperty = getProject().getProperties().get("konanMetaVersion");
if (konanMetaVersionProperty == null) {
return "MetaVersion.DEV";
}
return "MetaVersion." + konanMetaVersionProperty.toString().toUpperCase();
}
private final static Pattern versionPattern = Pattern.compile(
"^(\\d+)\\.(\\d+)(?:\\.(\\d+))?(?:-M(\\p{Digit}))?(?:-(\\p{Alpha}\\p{Alnum}*))?(?:-(\\d+))?$"
);
@TaskAction
public void generateVersion() {
Matcher matcher = versionPattern.matcher(getKonanVersion());
if (!matcher.matches()) {
throw new IllegalArgumentException("Cannot parse Kotlin/Native version: $konanVersion");
}
int major = Integer.parseInt(matcher.group(1));
int minor = Integer.parseInt(matcher.group(2));
String maintenanceStr = matcher.group(3);
int maintenance = maintenanceStr != null ? Integer.parseInt(maintenanceStr) : 0;
String milestoneStr = matcher.group(4);
int milestone = milestoneStr != null ? Integer.parseInt(milestoneStr) : -1;
String buildNumber = getBuildNumber();
getProject().getLogger().info("BUILD_NUMBER: " + getBuildNumber());
int build = -1;
if (buildNumber != null) {
String[] buildNumberSplit = buildNumber.split("-");
build = Integer.parseInt(buildNumberSplit[buildNumberSplit.length - 1]); // //7-dev-buildcount
}
try (PrintWriter printWriter = new PrintWriter(getVersionFile())) {
printWriter.println(
"package org.jetbrains.kotlin.konan\n" +
"\n" +
"internal val currentCompilerVersion: CompilerVersion =\n" +
" CompilerVersionImpl(\n" +
getMeta() + ", " + major + ", " + minor + ",\n" +
maintenance + ", " + milestone + ", "+ build + ")\n" +
"\n" +
"val CompilerVersion.Companion.CURRENT: CompilerVersion\n" +
" get() = currentCompilerVersion"
);
} catch (FileNotFoundException e) {
throw new IllegalStateException(e);
}
}
}
+24
View File
@@ -0,0 +1,24 @@
pluginManagement {
val rootProperties = java.util.Properties().apply {
rootDir.resolve("../gradle.properties").reader().use(::load)
}
val kotlinCompilerRepo: String by rootProperties
val kotlinVersion: String by rootProperties
repositories {
maven(kotlinCompilerRepo)
maven("https://cache-redirector.jetbrains.com/maven-central")
mavenCentral()
}
resolutionStrategy {
eachPlugin {
if (requested.id.id == "kotlin") {
useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
}
}
}
}
rootProject.name = "kotlin-native-shared"
@@ -0,0 +1,23 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.library.*
const val KLIB_PROPERTY_LINKED_OPTS = "linkerOpts"
const val KLIB_PROPERTY_INCLUDED_HEADERS = "includedHeaders"
interface TargetedLibrary {
val targetList: List<String>
val includedPaths: List<String>
}
interface BitcodeLibrary : TargetedLibrary {
val bitcodePaths: List<String>
}
interface KonanLibrary : BitcodeLibrary, KotlinLibrary {
val linkerOpts: List<String>
}
val KonanLibrary.includedHeaders
get() = manifestProperties.propertyList(KLIB_PROPERTY_INCLUDED_HEADERS, escapeInQuotes = true)
@@ -0,0 +1,50 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.library.IrKotlinLibraryLayout
import org.jetbrains.kotlin.library.KotlinLibraryLayout
import org.jetbrains.kotlin.library.MetadataKotlinLibraryLayout
interface TargetedKotlinLibraryLayout : KotlinLibraryLayout {
val target: KonanTarget?
// This is a default implementation. Can't make it an assignment.
get() = null
val targetsDir
get() = File(componentDir, "targets")
val targetDir
get() = File(targetsDir, target!!.visibleName)
val includedDir
get() = File(targetDir, "included")
}
interface BitcodeKotlinLibraryLayout : TargetedKotlinLibraryLayout, KotlinLibraryLayout {
val kotlinDir
get() = File(targetDir, "kotlin")
val nativeDir
get() = File(targetDir, "native")
// TODO: Experiment with separate bitcode files.
// Per package or per class.
val mainBitcodeFile
get() = File(kotlinDir, "program.kt.bc")
val mainBitcodeFileName
get() = mainBitcodeFile.path
}
interface KonanLibraryLayout : MetadataKotlinLibraryLayout, BitcodeKotlinLibraryLayout, IrKotlinLibraryLayout
@@ -0,0 +1,20 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.library.BaseWriter
import org.jetbrains.kotlin.library.IrWriter
import org.jetbrains.kotlin.library.MetadataWriter
interface TargetedWriter {
fun addIncludedBinary(library: String)
}
interface BitcodeWriter : TargetedWriter {
fun addNativeBitcode(library: String)
}
interface KonanLibraryWriter : MetadataWriter, BaseWriter, IrWriter, BitcodeWriter
@@ -0,0 +1,102 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.CompilerVersion
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
import org.jetbrains.kotlin.konan.library.impl.createKonanLibraryComponents
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.createKotlinLibrary
import org.jetbrains.kotlin.util.DummyLogger
import org.jetbrains.kotlin.util.Logger
interface SearchPathResolverWithTarget<L: KotlinLibrary>: SearchPathResolverWithAttributes<L> {
val target: KonanTarget
}
fun defaultResolver(
repositories: List<String>,
target: KonanTarget,
distribution: Distribution,
compatibleCompilerVersions: List<CompilerVersion> = emptyList()
): SearchPathResolverWithTarget<KonanLibrary> = defaultResolver(repositories, emptyList(), target, distribution, compatibleCompilerVersions)
fun defaultResolver(
repositories: List<String>,
directLibs: List<String>,
target: KonanTarget,
distribution: Distribution,
compatibleCompilerVersions: List<CompilerVersion> = emptyList(),
logger: Logger = DummyLogger,
skipCurrentDir: Boolean = false
): SearchPathResolverWithTarget<KonanLibrary> = KonanLibraryProperResolver(
repositories,
directLibs,
target,
listOf(KotlinAbiVersion.CURRENT),
compatibleCompilerVersions,
distribution.klib,
distribution.localKonanDir.absolutePath,
skipCurrentDir,
logger
)
fun resolverByName(
repositories: List<String>,
directLibs: List<String> = emptyList(),
distributionKlib: String? = null,
localKotlinDir: String? = null,
skipCurrentDir: Boolean = false,
logger: Logger
): SearchPathResolver<KotlinLibrary> =
object : KotlinLibrarySearchPathResolver<KotlinLibrary>(
repositories,
directLibs,
distributionKlib,
localKotlinDir,
skipCurrentDir,
logger
) {
override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKonanLibraryComponents(file, null, isDefault)
}
internal class KonanLibraryProperResolver(
repositories: List<String>,
directLibs: List<String>,
override val target: KonanTarget,
knownAbiVersions: List<KotlinAbiVersion>?,
knownCompilerVersions: List<CompilerVersion>?,
distributionKlib: String?,
localKonanDir: String?,
skipCurrentDir: Boolean,
override val logger: Logger
) : KotlinLibraryProperResolverWithAttributes<KonanLibrary>(
repositories, directLibs,
knownAbiVersions,
knownCompilerVersions,
distributionKlib,
localKonanDir,
skipCurrentDir,
logger,
listOf(KLIB_INTEROP_IR_PROVIDER_IDENTIFIER)
), SearchPathResolverWithTarget<KonanLibrary>
{
override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKonanLibraryComponents(file, target, isDefault)
override val distPlatformHead: File?
get() = distributionKlib?.File()?.child("platform")?.child(target.visibleName)
override fun libraryMatch(candidate: KonanLibrary, unresolved: UnresolvedLibrary): Boolean {
val resolverTarget = this.target
val candidatePath = candidate.libraryFile.absolutePath
if (!candidate.targetList.contains(resolverTarget.visibleName)) {
logger.warning("skipping $candidatePath. The target doesn't match. Expected '$resolverTarget', found ${candidate.targetList}")
return false
}
return super.libraryMatch(candidate, unresolved)
}
}
@@ -0,0 +1,40 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.library.BitcodeWriter
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.BitcodeKotlinLibraryLayout
import org.jetbrains.kotlin.konan.library.TargetedKotlinLibraryLayout
open class TargetedWriterImpl(val targetLayout: TargetedKotlinLibraryLayout) {
init {
targetLayout.targetDir.mkdirs()
targetLayout.includedDir.mkdirs()
}
fun addIncludedBinary(library: String) {
val basename = File(library).name
File(library).copyTo(File(targetLayout.includedDir, basename))
}
}
class BitcodeWriterImpl(
libraryLayout: BitcodeKotlinLibraryLayout
) : BitcodeWriter, TargetedWriterImpl(libraryLayout) {
val bitcodeLayout = libraryLayout
init {
bitcodeLayout.kotlinDir.mkdirs()
bitcodeLayout.nativeDir.mkdirs()
}
override fun addNativeBitcode(library: String) {
val basename = File(library).name
File(library).copyTo(File(bitcodeLayout.nativeDir, basename))
}
}
@@ -0,0 +1,115 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.*
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.*
open class TargetedLibraryImpl(
private val access: TargetedLibraryAccess<TargetedKotlinLibraryLayout>,
private val base: BaseKotlinLibrary
) : TargetedLibrary, BaseKotlinLibrary by base {
private val target: KonanTarget? get() = access.target
override val targetList: List<String>
get() = nativeTargets.ifEmpty {
// TODO: We have a choice: either assume it is the CURRENT TARGET
// or a list of ALL KNOWN targets.
listOfNotNull(access.target?.visibleName)
}
override val manifestProperties: Properties by lazy {
val properties = access.inPlace {
it.manifestFile.loadProperties()
}
target?.let { substitute(properties, defaultTargetSubstitutions(it)) }
properties
}
override val includedPaths: List<String>
get() = access.realFiles {
it.includedDir.listFilesOrEmpty.map { it.absolutePath }
}
}
open class BitcodeLibraryImpl(
private val access: BitcodeLibraryAccess<BitcodeKotlinLibraryLayout>,
targeted: TargetedLibrary
) : BitcodeLibrary, TargetedLibrary by targeted {
override val bitcodePaths: List<String>
get() = access.realFiles { it: BitcodeKotlinLibraryLayout ->
(it.kotlinDir.listFilesOrEmpty + it.nativeDir.listFilesOrEmpty).map { it.absolutePath }
}
}
class KonanLibraryImpl(
targeted: TargetedLibraryImpl,
metadata: MetadataLibraryImpl,
ir: IrLibraryImpl,
bitcode: BitcodeLibraryImpl
) : KonanLibrary,
BaseKotlinLibrary by targeted,
MetadataLibrary by metadata,
IrLibrary by ir,
BitcodeLibrary by bitcode {
override val linkerOpts: List<String>
get() = manifestProperties.propertyList(KLIB_PROPERTY_LINKED_OPTS, escapeInQuotes = true)
}
fun createKonanLibrary(
libraryFile: File,
component: String,
target: KonanTarget? = null,
isDefault: Boolean = false
): KonanLibrary {
val baseAccess = BaseLibraryAccess<KotlinLibraryLayout>(libraryFile, component)
val targetedAccess = TargetedLibraryAccess<TargetedKotlinLibraryLayout>(libraryFile, component, target)
val metadataAccess = MetadataLibraryAccess<MetadataKotlinLibraryLayout>(libraryFile, component)
val irAccess = IrLibraryAccess<IrKotlinLibraryLayout>(libraryFile, component)
val bitcodeAccess = BitcodeLibraryAccess<BitcodeKotlinLibraryLayout>(libraryFile, component, target)
val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
val targeted = TargetedLibraryImpl(targetedAccess, base)
val metadata = MetadataLibraryImpl(metadataAccess)
val ir = IrMonoliticLibraryImpl(irAccess)
val bitcode = BitcodeLibraryImpl(bitcodeAccess, targeted)
return KonanLibraryImpl(targeted, metadata, ir, bitcode)
}
fun createKonanLibraryComponents(
libraryFile: File,
target: KonanTarget? = null,
isDefault: Boolean = true
) : List<KonanLibrary> {
val baseAccess = BaseLibraryAccess<KotlinLibraryLayout>(libraryFile, null)
val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
return base.componentList.map {
createKonanLibrary(libraryFile, it, target, isDefault)
}
}
@@ -0,0 +1,61 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.*
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.*
import java.nio.file.FileSystem
open class TargetedLibraryLayoutImpl(klib: File, component: String, override val target: KonanTarget?) :
KotlinLibraryLayoutImpl(klib, component), TargetedKotlinLibraryLayout {
override val extractingToTemp: TargetedKotlinLibraryLayout by lazy {
ExtractingTargetedLibraryImpl(this)
}
override fun directlyFromZip(zipFileSystem: FileSystem): TargetedKotlinLibraryLayout =
FromZipTargetedLibraryImpl(this, zipFileSystem)
}
class BitcodeLibraryLayoutImpl(klib: File, component: String, target: KonanTarget?) :
TargetedLibraryLayoutImpl(klib, component, target), BitcodeKotlinLibraryLayout {
override val extractingToTemp: BitcodeKotlinLibraryLayout by lazy {
ExtractingBitcodeLibraryImpl(this)
}
override fun directlyFromZip(zipFileSystem: FileSystem): BitcodeKotlinLibraryLayout =
FromZipBitcodeLibraryImpl(this, zipFileSystem)
}
open class TargetedLibraryAccess<L : KotlinLibraryLayout>(klib: File, component: String, val target: KonanTarget?) :
BaseLibraryAccess<L>(klib, component) {
override val layout = TargetedLibraryLayoutImpl(klib, component, target)
}
open class BitcodeLibraryAccess<L : KotlinLibraryLayout>(klib: File, component: String, target: KonanTarget?) :
TargetedLibraryAccess<L>(klib, component, target) {
override val layout = BitcodeLibraryLayoutImpl(klib, component, target)
}
private open class FromZipTargetedLibraryImpl(zipped: TargetedLibraryLayoutImpl, zipFileSystem: FileSystem) :
FromZipBaseLibraryImpl(zipped, zipFileSystem), TargetedKotlinLibraryLayout
private class FromZipBitcodeLibraryImpl(zipped: BitcodeLibraryLayoutImpl, zipFileSystem: FileSystem) :
FromZipTargetedLibraryImpl(zipped, zipFileSystem), BitcodeKotlinLibraryLayout
open class ExtractingTargetedLibraryImpl(zipped: TargetedLibraryLayoutImpl) :
ExtractingKotlinLibraryLayout(zipped),
TargetedKotlinLibraryLayout {
override val includedDir: File by lazy { zipped.extractDir(zipped.includedDir) }
}
class ExtractingBitcodeLibraryImpl(zipped: BitcodeLibraryLayoutImpl) :
ExtractingTargetedLibraryImpl(zipped), BitcodeKotlinLibraryLayout {
override val kotlinDir: File by lazy { zipped.extractDir(zipped.kotlinDir) }
override val nativeDir: File by lazy { zipped.extractDir(zipped.nativeDir) }
}
@@ -0,0 +1,87 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.library.BitcodeWriter
import org.jetbrains.kotlin.konan.library.KonanLibraryWriter
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.*
class KonanLibraryLayoutForWriter(
override val libDir: File,
override val target: KonanTarget
) : KonanLibraryLayout, KotlinLibraryLayoutForWriter(libDir)
/**
* Requires non-null [target].
*/
class KonanLibraryWriterImpl(
libDir: File,
moduleName: String,
versions: KotlinLibraryVersioning,
target: KonanTarget,
builtInsPlatform: BuiltInsPlatform,
nopack: Boolean = false,
shortName: String? = null,
val layout: KonanLibraryLayoutForWriter = KonanLibraryLayoutForWriter(libDir, target),
base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, builtInsPlatform, listOf(target.visibleName), nopack, shortName),
bitcode: BitcodeWriter = BitcodeWriterImpl(layout),
metadata: MetadataWriter = MetadataWriterImpl(layout),
ir: IrWriter = IrMonoliticWriterImpl(layout)
) : BaseWriter by base, BitcodeWriter by bitcode, MetadataWriter by metadata, IrWriter by ir, KonanLibraryWriter
fun buildLibrary(
natives: List<String>,
included: List<String>,
linkDependencies: List<KonanLibrary>,
metadata: SerializedMetadata,
ir: SerializedIrModule?,
versions: KotlinLibraryVersioning,
target: KonanTarget,
output: String,
moduleName: String,
nopack: Boolean,
shortName: String?,
manifestProperties: Properties?,
dataFlowGraph: ByteArray?
): KonanLibraryLayout {
val library = KonanLibraryWriterImpl(
File(output),
moduleName,
versions,
target,
BuiltInsPlatform.NATIVE,
nopack,
shortName
)
library.addMetadata(metadata)
if (ir != null) {
library.addIr(ir)
}
natives.forEach {
library.addNativeBitcode(it)
}
included.forEach {
library.addIncludedBinary(it)
}
manifestProperties?.let { library.addManifestAddend(it) }
library.addLinkDependencies(linkDependencies)
dataFlowGraph?.let { library.addDataFlowGraph(it) }
library.commit()
return library.layout
}
@@ -0,0 +1,52 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
/**
* This is a common ancestor of all Kotlin/Native exceptions.
*/
open class KonanException(message: String = "", cause: Throwable? = null) : Exception(message, cause)
/**
* An error occurred during external tool invocation. Such as non-zero exit code.
*/
class KonanExternalToolFailure(message: String, val toolName: String, cause: Throwable? = null) : KonanException(message, cause)
/**
* An exception indicating a failed attempt to access some parts of Xcode (e.g. get SDK paths or version).
*/
class MissingXcodeException(message: String, cause: Throwable? = null) : KonanException(message, cause)
/**
* Native exception handling in Kotlin: terminate, wrap, etc.
* Foreign exceptionMode mode is per library option: controlled by cinterop command-line option or def file property
* than stored in klib manifest and used by compiler to generate appropriate handler.
*/
class ForeignExceptionMode {
companion object {
val manifestKey = "foreignExceptionMode"
val default = Mode.TERMINATE
fun byValue(value: String?): Mode = value?.let {
Mode.values().find { it.value == value }
?: throw IllegalArgumentException("Illegal ForeignExceptionMode $value")
} ?: default
}
enum class Mode(val value: String) {
TERMINATE("terminate"),
OBJC_WRAP("objc-wrap")
}
}
@@ -0,0 +1,28 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
fun String.parseKonanAbiVersion(): KonanAbiVersion {
return KonanAbiVersion(this.toInt())
}
data class KonanAbiVersion(val version: Int) {
companion object {
val CURRENT = KonanAbiVersion(10)
}
override fun toString() = "$version"
}
@@ -0,0 +1,62 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.*
/**
* Creates and stores temporary compiler outputs
* If pathToTemporaryDir is given and is not empty then temporary outputs will be preserved
*/
class TempFiles(outputPath: String, pathToTemporaryDir: String? = null) {
private val outputName = File(outputPath).name
val deleteOnExit = pathToTemporaryDir == null || pathToTemporaryDir.isEmpty()
val nativeBinaryFile by lazy { create(outputName,".kt.bc") }
val cAdapterCpp by lazy { create("api", ".cpp") }
val cAdapterBitcode by lazy { create("api", ".bc") }
val nativeBinaryFileName get() = nativeBinaryFile.absolutePath
val cAdapterCppName get() = cAdapterCpp.absolutePath
val cAdapterBitcodeName get() = cAdapterBitcode.absolutePath
private val dir by lazy {
if (deleteOnExit) {
createTempDir("konan_temp").deleteOnExit()
} else {
createDirForTemporaryFiles(pathToTemporaryDir!!)
}
}
private fun createDirForTemporaryFiles(path: String): File {
if (File(path).isFile) {
throw IllegalArgumentException("Given file is not a directory: $path")
}
return File(path).apply {
if (!exists) { mkdirs() }
}
}
/**
* Create file named {name}{suffix} inside temporary dir
*/
fun create(prefix: String, suffix: String = ""): File =
File(dir, "$prefix$suffix").also {
if (deleteOnExit) it.deleteOnExit()
}
}
@@ -0,0 +1,128 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.exec
import org.jetbrains.kotlin.konan.KonanExternalToolFailure
import java.io.BufferedReader
import java.io.InputStreamReader
import java.lang.ProcessBuilder.Redirect
open class Command(initialCommand: List<String>) {
constructor(tool: String) : this(listOf(tool))
constructor(vararg command: String) : this(command.toList<String>())
protected val command = initialCommand.toMutableList()
val argsWithExecutable: List<String> = command
val args: List<String>
get() = command.drop(1)
operator fun String.unaryPlus(): Command {
command += this
return this@Command
}
operator fun List<String>.unaryPlus(): Command {
command.addAll(this)
return this@Command
}
var logger: ((() -> String)->Unit)? = null
private var stdError: List<String> = emptyList()
fun logWith(newLogger: ((() -> String)->Unit)): Command {
logger = newLogger
return this
}
open fun runProcess(): Int {
stdError = emptyList()
val builder = ProcessBuilder(command)
builder.redirectOutput(Redirect.INHERIT)
builder.redirectInput(Redirect.INHERIT)
val process = builder.start()
val reader = BufferedReader(InputStreamReader(process.errorStream))
stdError = reader.readLines()
val exitCode = process.waitFor()
return exitCode
}
open fun execute() {
log()
val code = runProcess()
handleExitCode(code, stdError)
}
/**
* If withErrors is true then output from error stream will be added
*/
fun getOutputLines(withErrors: Boolean = false): List<String> =
getResult(withErrors, handleError = true).outputLines
fun getResult(withErrors: Boolean, handleError: Boolean = false): Result {
log()
val outputFile = createTempFile()
outputFile.deleteOnExit()
try {
val builder = ProcessBuilder(command)
builder.redirectInput(Redirect.INHERIT)
builder.redirectError(Redirect.INHERIT)
builder.redirectOutput(Redirect.to(outputFile))
.redirectErrorStream(withErrors)
// Note: getting process output could be done without redirecting to temporary file,
// however this would require managing a thread to read `process.inputStream` because
// it may have limited capacity.
val process = builder.start()
val code = process.waitFor()
if (handleError) handleExitCode(code, outputFile.readLines())
return Result(code, outputFile.readLines())
} finally {
outputFile.delete()
}
}
class Result(val exitCode: Int, val outputLines: List<String>)
private fun handleExitCode(code: Int, output: List<String> = emptyList()) {
if (code != 0) throw KonanExternalToolFailure("""
The ${command[0]} command returned non-zero exit code: $code.
output:
""".trimIndent() + "\n${output.joinToString("\n")}", command[0])
// Show warnings in case of success linkage.
if (stdError.isNotEmpty()) {
stdError.joinToString("\n").also { message ->
logger?.let { it { message } } ?: println(message)
}
}
}
private fun log() {
if (logger != null) logger!! { command.joinToString(" ") }
}
}
@@ -0,0 +1,29 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.file
val String.isJavaScript
get() = this.endsWith(".js")
val String.isUnixStaticLib
get() = this.endsWith(".a")
val String.isWindowsStaticLib
get() = this.endsWith(".lib")
val String.isBitcode
get() = this.endsWith(".bc")
@@ -0,0 +1,103 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed -> in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.util.InternalServer
import kotlin.math.max
class AppleConfigurablesImpl(
target: KonanTarget,
properties: Properties,
baseDir: String?
) : AppleConfigurables, KonanPropertiesLoader(target, properties, baseDir) {
private val sdkDependency = this.targetSysRoot!!
private val toolchainDependency = this.targetToolchain!!
private val xcodeAddonDependency = this.additionalToolsDir!!
override val absoluteTargetSysRoot: String get() = when (val provider = xcodePartsProvider) {
is XcodePartsProvider.Local -> when (target) {
KonanTarget.MACOS_X64 -> provider.xcode.macosxSdk
KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64 -> provider.xcode.iphoneosSdk
KonanTarget.IOS_X64 -> provider.xcode.iphonesimulatorSdk
KonanTarget.TVOS_ARM64 -> provider.xcode.appletvosSdk
KonanTarget.TVOS_X64 -> provider.xcode.appletvsimulatorSdk
KonanTarget.WATCHOS_ARM64, KonanTarget.WATCHOS_ARM32 -> provider.xcode.watchosSdk
KonanTarget.WATCHOS_X64, KonanTarget.WATCHOS_X86 -> provider.xcode.watchsimulatorSdk
else -> error(target)
}
XcodePartsProvider.InternalServer -> absolute(sdkDependency)
}
override val absoluteTargetToolchain: String get() = when (val provider = xcodePartsProvider) {
is XcodePartsProvider.Local -> provider.xcode.toolchain
XcodePartsProvider.InternalServer -> absolute(toolchainDependency)
}
override val absoluteAdditionalToolsDir: String get() = when (val provider = xcodePartsProvider) {
is XcodePartsProvider.Local -> provider.xcode.additionalTools
XcodePartsProvider.InternalServer -> absolute(additionalToolsDir)
}
override val dependencies get() = super.dependencies + when (xcodePartsProvider) {
is XcodePartsProvider.Local -> emptyList()
XcodePartsProvider.InternalServer -> listOf(sdkDependency, toolchainDependency, xcodeAddonDependency)
}
private val xcodePartsProvider by lazy {
if (InternalServer.isAvailable) {
XcodePartsProvider.InternalServer
} else {
val xcode = Xcode.current
if (properties.getProperty("ignoreXcodeVersionCheck") != "true") {
properties.getProperty("minimalXcodeVersion")?.let { minimalXcodeVersion ->
val currentXcodeVersion = xcode.version
checkXcodeVersion(minimalXcodeVersion, currentXcodeVersion)
}
}
XcodePartsProvider.Local(xcode)
}
}
private fun checkXcodeVersion(minimalVersion: String, currentVersion: String) {
// Xcode versions contain only numbers (even betas).
// But we still split by '-' and whitespaces to take into account versions like 11.2-beta.
val minimalVersionParts = minimalVersion.split("(\\s+|\\.|-)".toRegex()).map { it.toIntOrNull() ?: 0 }
val currentVersionParts = currentVersion.split("(\\s+|\\.|-)".toRegex()).map { it.toIntOrNull() ?: 0 }
val size = max(minimalVersionParts.size, currentVersionParts.size)
for (i in 0 until size) {
val currentPart = currentVersionParts.getOrElse(i) { 0 }
val minimalPart = minimalVersionParts.getOrElse(i) { 0 }
when {
currentPart > minimalPart -> return
currentPart < minimalPart ->
error("Unsupported Xcode version $currentVersion, minimal supported version is $minimalVersion.")
}
}
}
private sealed class XcodePartsProvider {
class Local(val xcode: Xcode) : XcodePartsProvider()
object InternalServer : XcodePartsProvider()
}
}
@@ -0,0 +1,232 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed -> in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.file.File
internal object Android {
const val API = "21"
private val architectureMap = mapOf(
KonanTarget.ANDROID_X86 to "x86",
KonanTarget.ANDROID_X64 to "x86_64",
KonanTarget.ANDROID_ARM32 to "arm",
KonanTarget.ANDROID_ARM64 to "arm64"
)
fun architectureDirForTarget(target: KonanTarget) =
"android-${API}/arch-${architectureMap.getValue(target)}"
}
class ClangArgs(private val configurables: Configurables) : Configurables by configurables {
val targetArg = if (configurables is TargetableConfigurables)
configurables.targetArg
else null
private val osVersionMin: String
get() {
require(configurables is AppleConfigurables)
return configurables.osVersionMin
}
val specificClangArgs: List<String>
get() {
val result = when (target) {
KonanTarget.LINUX_X64 ->
listOf("--sysroot=$absoluteTargetSysRoot") +
if (target != host) listOf("-target", targetArg!!) else emptyList()
KonanTarget.LINUX_ARM32_HFP ->
listOf("-target", targetArg!!,
"-mfpu=vfp", "-mfloat-abi=hard",
"--sysroot=$absoluteTargetSysRoot",
// TODO: those two are hacks.
"-I$absoluteTargetSysRoot/usr/include/c++/4.8.3",
"-I$absoluteTargetSysRoot/usr/include/c++/4.8.3/arm-linux-gnueabihf")
KonanTarget.LINUX_ARM64 ->
listOf("-target", targetArg!!,
"--sysroot=$absoluteTargetSysRoot",
"-I$absoluteTargetSysRoot/usr/include/c++/7",
"-I$absoluteTargetSysRoot/usr/include/c++/7/aarch64-linux-gnu")
KonanTarget.LINUX_MIPS32 ->
listOf("-target", targetArg!!,
"--sysroot=$absoluteTargetSysRoot",
"-I$absoluteTargetSysRoot/usr/include/c++/4.9.4",
"-I$absoluteTargetSysRoot/usr/include/c++/4.9.4/mips-unknown-linux-gnu")
KonanTarget.LINUX_MIPSEL32 ->
listOf("-target", targetArg!!,
"--sysroot=$absoluteTargetSysRoot",
"-I$absoluteTargetSysRoot/usr/include/c++/4.9.4",
"-I$absoluteTargetSysRoot/usr/include/c++/4.9.4/mipsel-unknown-linux-gnu")
KonanTarget.MINGW_X64, KonanTarget.MINGW_X86 ->
listOf("-target", targetArg!!, "--sysroot=$absoluteTargetSysRoot", "-Xclang", "-flto-visibility-public-std")
KonanTarget.MACOS_X64 ->
listOf("--sysroot=$absoluteTargetSysRoot", "-mmacosx-version-min=$osVersionMin")
KonanTarget.IOS_ARM32 ->
listOf("-stdlib=libc++", "-arch", "armv7", "-isysroot", absoluteTargetSysRoot, "-miphoneos-version-min=$osVersionMin")
KonanTarget.IOS_ARM64 ->
listOf("-stdlib=libc++", "-arch", "arm64", "-isysroot", absoluteTargetSysRoot, "-miphoneos-version-min=$osVersionMin")
KonanTarget.IOS_X64 ->
listOf("-stdlib=libc++", "-isysroot", absoluteTargetSysRoot, "-miphoneos-version-min=$osVersionMin")
KonanTarget.TVOS_ARM64 ->
listOf("-stdlib=libc++", "-arch", "arm64", "-isysroot", absoluteTargetSysRoot, "-mtvos-version-min=$osVersionMin")
KonanTarget.TVOS_X64 ->
listOf("-stdlib=libc++", "-isysroot", absoluteTargetSysRoot, "-mtvos-simulator-version-min=$osVersionMin")
KonanTarget.WATCHOS_ARM64,
KonanTarget.WATCHOS_ARM32 ->
listOf("-stdlib=libc++", "-arch", "armv7k", "-isysroot", absoluteTargetSysRoot, "-mwatchos-version-min=$osVersionMin")
KonanTarget.WATCHOS_X86 ->
listOf("-stdlib=libc++", "-arch", "i386", "-isysroot", absoluteTargetSysRoot, "-mwatchos-simulator-version-min=$osVersionMin")
KonanTarget.WATCHOS_X64 -> TODO("implement me")
KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64,
KonanTarget.ANDROID_X86, KonanTarget.ANDROID_X64 -> {
val clangTarget = targetArg!!
val architectureDir = Android.architectureDirForTarget(target)
val toolchainSysroot = "$absoluteTargetToolchain/sysroot"
listOf("-target", clangTarget,
"-D__ANDROID_API__=${Android.API}",
"--sysroot=$absoluteTargetSysRoot/$architectureDir",
"-I$toolchainSysroot/usr/include/c++/v1",
"-I$toolchainSysroot/usr/include",
"-I$toolchainSysroot/usr/include/$clangTarget")
}
// By default WASM target forces `hidden` visibility which causes linkage problems.
KonanTarget.WASM32 ->
listOf("-target", targetArg!!,
"-fno-rtti",
"-fno-exceptions",
"-fvisibility=default",
"-D_LIBCPP_ABI_VERSION=2",
"-D_LIBCPP_NO_EXCEPTIONS=1",
"-nostdinc",
"-Xclang", "-nobuiltininc",
"-Xclang", "-nostdsysteminc",
"-Xclang", "-isystem$absoluteTargetSysRoot/include/libcxx",
"-Xclang", "-isystem$absoluteTargetSysRoot/lib/libcxxabi/include",
"-Xclang", "-isystem$absoluteTargetSysRoot/include/compat",
"-Xclang", "-isystem$absoluteTargetSysRoot/include/libc")
is KonanTarget.ZEPHYR ->
listOf("-target", targetArg!!,
"-fno-rtti",
"-fno-exceptions",
"-fno-asynchronous-unwind-tables",
"-fno-pie",
"-fno-pic",
"-fshort-enums",
"-nostdinc",
// TODO: make it a libGcc property?
// We need to get rid of wasm sysroot first.
"-isystem $targetToolchain/../lib/gcc/arm-none-eabi/7.2.1/include",
"-isystem $targetToolchain/../lib/gcc/arm-none-eabi/7.2.1/include-fixed",
"-isystem$absoluteTargetSysRoot/include/libcxx",
"-isystem$absoluteTargetSysRoot/include/libc"
) +
(configurables as ZephyrConfigurables).constructClangArgs()
}
return result
}
val clangArgsSpecificForKonanSources
get() = runtimeDefinitions.map { "-D$it" }
private val host = HostManager.host
private val binDir = when (host) {
KonanTarget.LINUX_X64 -> "$absoluteTargetToolchain/bin"
KonanTarget.MINGW_X64 -> "$absoluteTargetToolchain/bin"
KonanTarget.MACOS_X64 -> "$absoluteTargetToolchain/usr/bin"
else -> throw TargetSupportException("Unexpected host platform")
}
private val extraHostClangArgs =
if (configurables is GccConfigurables) {
listOf("--gcc-toolchain=${configurables.absoluteGccToolchain}")
} else {
emptyList()
}
val commonClangArgs = listOf("-B$binDir", "-fno-stack-protector") + extraHostClangArgs
val clangPaths = listOf("$absoluteLlvmHome/bin", binDir)
private val jdkDir by lazy {
val home = File.javaHome.absoluteFile
if (home.child("include").exists)
home.absolutePath
else
home.parentFile.absolutePath
}
val hostCompilerArgsForJni = listOf("", HostManager.jniHostPlatformIncludeDir).map { "-I$jdkDir/include/$it" }.toTypedArray()
val clangArgs = (commonClangArgs + specificClangArgs).toTypedArray()
val clangArgsForKonanSources =
clangArgs + clangArgsSpecificForKonanSources
val targetLibclangArgs: List<String> =
// libclang works not exactly the same way as the clang binary and
// (in particular) uses different default header search path.
// See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html
// We workaround the problem with -isystem flag below.
listOf("-isystem", "$absoluteLlvmHome/lib/clang/$llvmVersion/include", *clangArgs)
val targetClangCmd
= listOf("${absoluteLlvmHome}/bin/clang") + clangArgs
val targetClangXXCmd
= listOf("${absoluteLlvmHome}/bin/clang++") + clangArgs
fun clangC(vararg userArgs: String) = targetClangCmd + userArgs.asList()
fun clangCXX(vararg userArgs: String) = targetClangXXCmd + userArgs.asList()
companion object {
@JvmStatic
fun filterGradleNativeSoftwareFlags(args: MutableList<String>) {
args.remove("/usr/include") // HACK: over gradle-4.4.
args.remove("-nostdinc") // HACK: over gradle-5.1.
when (HostManager.host) {
KonanTarget.LINUX_X64 -> args.remove("/usr/include/x86_64-linux-gnu") // HACK: over gradle-4.4.
KonanTarget.MACOS_X64 -> {
val indexToRemove = args.indexOf(args.find { it.contains("MacOSX.platform")}) // HACK: over gradle-4.7.
if (indexToRemove != -1) {
args.removeAt(indexToRemove - 1) // drop -I.
args.removeAt(indexToRemove - 1) // drop /Application/Xcode.app/...
}
}
}
}
}
}
@@ -0,0 +1,90 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed -> in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.properties.*
interface ClangFlags : TargetableExternalStorage {
val clangFlags get() = targetList("clangFlags")
val clangNooptFlags get() = targetList("clangNooptFlags")
val clangOptFlags get() = targetList("clangOptFlags")
val clangDebugFlags get() = targetList("clangDebugFlags")
val clangDynamicFlags get() = targetList("clangDynamicFlags")
}
interface LldFlags : TargetableExternalStorage {
val lldFlags get() = targetList("lld")
}
interface Configurables : TargetableExternalStorage {
val target: KonanTarget
val llvmHome get() = hostString("llvmHome")
val llvmVersion get() = hostString("llvmVersion")
val libffiDir get() = hostString("libffiDir")
// TODO: Delegate to a map?
val linkerOptimizationFlags get() = targetList("linkerOptimizationFlags")
val linkerKonanFlags get() = targetList("linkerKonanFlags")
val mimallocLinkerDependencies get() = targetList("mimallocLinkerDependencies")
val linkerNoDebugFlags get() = targetList("linkerNoDebugFlags")
val linkerDynamicFlags get() = targetList("linkerDynamicFlags")
val targetSysRoot get() = targetString("targetSysRoot")
// Notice: these ones are host-target.
val targetToolchain get() = hostTargetString("targetToolchain")
val absoluteTargetSysRoot get() = absolute(targetSysRoot)
val absoluteTargetToolchain get() = absolute(targetToolchain)
val absoluteLlvmHome get() = absolute(llvmHome)
val runtimeDefinitions get() = targetList("runtimeDefinitions")
}
interface TargetableConfigurables : Configurables {
val targetArg get() = targetString("quadruple")
}
interface AppleConfigurables : Configurables, ClangFlags {
val arch get() = targetString("arch")!!
val osVersionMin get() = targetString("osVersionMin")!!
val osVersionMinFlagLd get() = targetString("osVersionMinFlagLd")!!
val additionalToolsDir get() = hostString("additionalToolsDir")
val absoluteAdditionalToolsDir get() = absolute(additionalToolsDir)
}
interface MingwConfigurables : TargetableConfigurables, ClangFlags
interface GccConfigurables : TargetableConfigurables, ClangFlags {
val gccToolchain get() = hostString("gccToolchain")
val absoluteGccToolchain get() = absolute(gccToolchain)
val libGcc get() = targetString("libGcc")!!
val dynamicLinker get() = targetString("dynamicLinker")!!
val abiSpecificLibraries get() = targetList("abiSpecificLibraries")
}
interface AndroidConfigurables : TargetableConfigurables, ClangFlags
interface WasmConfigurables : TargetableConfigurables, ClangFlags, LldFlags
interface ZephyrConfigurables : TargetableConfigurables, ClangFlags {
val boardSpecificClangFlags get() = targetList("boardSpecificClangFlags")
val targetCpu get() = targetString("targetCpu")
val targetAbi get() = targetString("targetAbi")
}
@@ -0,0 +1,23 @@
package org.jetbrains.kotlin.konan.target
fun ZephyrConfigurables.constructClangArgs(): List<String> = mutableListOf<String>().apply {
targetCpu?.let {
add("-mcpu=$it")
}
targetAbi?.let {
add("-mabi=$it")
}
addAll(boardSpecificClangFlags)
}
fun ZephyrConfigurables.constructClangCC1Args(): List<String> = mutableListOf<String>().apply {
addAll("-cc1 -emit-obj -disable-llvm-optzns -x ir -fdata-sections -ffunction-sections".split(" "))
targetCpu?.let {
addAll(listOf("-target-abi", it))
}
targetAbi?.let {
addAll(listOf("-target-cpu", it))
}
addAll(boardSpecificClangFlags)
}
@@ -0,0 +1,62 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed -> in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.properties.*
class GccConfigurablesImpl(target: KonanTarget, properties: Properties, baseDir: String?)
: GccConfigurables, KonanPropertiesLoader(target, properties, baseDir)
class AndroidConfigurablesImpl(target: KonanTarget, properties: Properties, baseDir: String?)
: AndroidConfigurables, KonanPropertiesLoader(target, properties, baseDir)
class MingwConfigurablesImpl(target: KonanTarget, properties: Properties, baseDir: String?)
: MingwConfigurables, KonanPropertiesLoader(target, properties, baseDir)
class WasmConfigurablesImpl(target: KonanTarget, properties: Properties, baseDir: String?)
: WasmConfigurables, KonanPropertiesLoader(target, properties, baseDir)
class ZephyrConfigurablesImpl(target: KonanTarget, properties: Properties, baseDir: String?)
: ZephyrConfigurables, KonanPropertiesLoader(target, properties, baseDir)
fun loadConfigurables(target: KonanTarget, properties: Properties, baseDir: String?): Configurables = when (target) {
KonanTarget.LINUX_X64, KonanTarget.LINUX_ARM32_HFP, KonanTarget.LINUX_ARM64,
KonanTarget.LINUX_MIPS32, KonanTarget.LINUX_MIPSEL32 ->
GccConfigurablesImpl(target, properties, baseDir)
KonanTarget.MACOS_X64,
KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64, KonanTarget.IOS_X64,
KonanTarget.TVOS_ARM64, KonanTarget.TVOS_X64,
KonanTarget.WATCHOS_ARM64, KonanTarget.WATCHOS_ARM32,
KonanTarget.WATCHOS_X64, KonanTarget.WATCHOS_X86 ->
AppleConfigurablesImpl(target, properties, baseDir)
KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64,
KonanTarget.ANDROID_X86, KonanTarget.ANDROID_X64 ->
AndroidConfigurablesImpl(target, properties, baseDir)
KonanTarget.MINGW_X64, KonanTarget.MINGW_X86 ->
MingwConfigurablesImpl(target, properties, baseDir)
KonanTarget.WASM32 ->
WasmConfigurablesImpl(target, properties, baseDir)
is KonanTarget.ZEPHYR ->
ZephyrConfigurablesImpl(target, properties, baseDir)
}
@@ -0,0 +1,93 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed -> in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.properties
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.Configurables
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.util.ArchiveType
import org.jetbrains.kotlin.konan.util.DependencyProcessor
import java.io.File
interface TargetableExternalStorage {
fun targetString(key: String): String?
fun targetList(key: String): List<String>
fun hostString(key: String): String?
fun hostList(key: String): List<String>
fun hostTargetString(key: String): String?
fun hostTargetList(key: String): List<String>
fun absolute(value: String?): String
fun downloadDependencies()
}
abstract class KonanPropertiesLoader(override val target: KonanTarget,
val properties: Properties,
private val baseDir: String? = null,
private val host: KonanTarget = HostManager.host) : Configurables {
private val predefinedLlvmDistributions: Set<String> =
properties.propertyList("predefinedLlvmDistributions").toSet()
private fun llvmDependencies(): List<String> {
// Store into variable to avoid repeated resolve.
val llvmHome = llvmHome
?: error("Undefined LLVM home!")
return when (llvmHome) {
in predefinedLlvmDistributions -> llvmHome
else -> null
}.let(::listOfNotNull)
}
open val dependencies: List<String>
get() = hostTargetList("dependencies") + llvmDependencies()
override fun downloadDependencies() {
dependencyProcessor!!.run()
}
// TODO: We may want to add caching to avoid repeated resolve.
override fun targetString(key: String): String?
= properties.targetString(key, target)
override fun targetList(key: String): List<String>
= properties.targetList(key, target)
override fun hostString(key: String): String?
= properties.hostString(key, host)
override fun hostList(key: String): List<String>
= properties.hostList(key, host)
override fun hostTargetString(key: String): String?
= properties.hostTargetString(key, target, host)
override fun hostTargetList(key: String): List<String>
= properties.hostTargetList(key, target, host)
override fun absolute(value: String?): String =
dependencyProcessor!!.resolve(value!!).absolutePath
private val dependencyProcessor by lazy {
baseDir?.let {
DependencyProcessor(
dependenciesRoot = File(baseDir),
properties = this,
archiveType = defaultArchiveTypeByHost(host)
)
}
}
}
private fun defaultArchiveTypeByHost(host: KonanTarget): ArchiveType = when (host) {
KonanTarget.LINUX_X64 -> ArchiveType.TAR_GZ
KonanTarget.MACOS_X64 -> ArchiveType.TAR_GZ
KonanTarget.MINGW_X64 -> ArchiveType.ZIP
else -> error("$host can't be a host platform!")
}
@@ -0,0 +1,32 @@
package org.jetbrains.kotlin.konan.target
// TODO: This all needs to go to konan.properties
fun KonanTarget.supportsCodeCoverage(): Boolean =
this == KonanTarget.MINGW_X64 ||
this == KonanTarget.LINUX_X64 ||
this == KonanTarget.MACOS_X64 ||
this == KonanTarget.IOS_X64
fun KonanTarget.supportsMimallocAllocator(): Boolean =
when(this) {
is KonanTarget.LINUX_X64 -> true
is KonanTarget.MINGW_X86 -> true
is KonanTarget.MINGW_X64 -> true
is KonanTarget.MACOS_X64 -> true
is KonanTarget.LINUX_ARM64 -> true
is KonanTarget.LINUX_ARM32_HFP -> true
is KonanTarget.ANDROID_X64 -> true
is KonanTarget.ANDROID_ARM64 -> true
is KonanTarget.IOS_ARM32 -> true
is KonanTarget.IOS_ARM64 -> true
is KonanTarget.IOS_X64 -> true
else -> false // watchOS/tvOS/android_x86/android_arm32 aren't tested; linux_mips32/linux_mipsel32 need linking with libatomic.
}
fun KonanTarget.supportsThreads(): Boolean =
when(this) {
is KonanTarget.WASM32 -> false
is KonanTarget.ZEPHYR -> false
else -> true
}
@@ -0,0 +1,517 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import java.lang.ProcessBuilder
import java.lang.ProcessBuilder.Redirect
import org.jetbrains.kotlin.konan.exec.Command
import org.jetbrains.kotlin.konan.file.*
typealias ObjectFile = String
typealias ExecutableFile = String
enum class LinkerOutputKind {
DYNAMIC_LIBRARY,
STATIC_LIBRARY,
EXECUTABLE
}
// Here we take somewhat unexpected approach - we create the thin
// library, and then repack it during post-link phase.
// This way we ensure .a inputs are properly processed.
private fun staticGnuArCommands(ar: String, executable: ExecutableFile,
objectFiles: List<ObjectFile>, libraries: List<String>) = when {
HostManager.hostIsMingw -> {
val temp = executable.replace('/', '\\') + "__"
val arWindows = ar.replace('/', '\\')
listOf(
Command(arWindows, "-rucT", temp).apply {
+objectFiles
+libraries
},
Command("cmd", "/c").apply {
+"(echo create $executable & echo addlib ${temp} & echo save & echo end) | $arWindows -M"
},
Command("cmd", "/c", "del", "/q", temp))
}
HostManager.hostIsLinux || HostManager.hostIsMac -> listOf(
Command(ar, "cqT", executable).apply {
+objectFiles
+libraries
},
Command("/bin/sh", "-c").apply {
+"printf 'create $executable\\naddlib $executable\\nsave\\nend' | $ar -M"
})
else -> TODO("Unsupported host ${HostManager.host}")
}
// Use "clang -v -save-temps" to write linkCommand() method
// for another implementation of this class.
abstract class LinkerFlags(val configurables: Configurables) {
protected val llvmBin = "${configurables.absoluteLlvmHome}/bin"
protected val llvmLib = "${configurables.absoluteLlvmHome}/lib"
open val useCompilerDriverAsLinker: Boolean get() = false // TODO: refactor.
/**
* Returns list of commands that produces final linker output.
*/
// TODO: Number of arguments is quite big. Better to pass args via object.
abstract fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command>
/**
* Returns list of commands that link object files into a single one.
* Pre-linkage is useful for hiding dependency symbols.
*/
open fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
error("Pre-link is unsupported for ${configurables.target}.")
abstract fun filterStaticLibraries(binaries: List<String>): List<String>
open fun linkStaticLibraries(binaries: List<String>): List<String> {
val libraries = filterStaticLibraries(binaries)
// Let's just pass them as absolute paths.
return libraries
}
protected open fun provideCompilerRtLibrary(libraryName: String): String? {
System.err.println("Can't provide $libraryName.")
return null
}
// Code coverage requires this library.
protected val profileLibrary: String? by lazy {
provideCompilerRtLibrary("profile")
}
}
class AndroidLinker(targetProperties: AndroidConfigurables)
: LinkerFlags(targetProperties), AndroidConfigurables by targetProperties {
private val clangQuad = when (targetProperties.targetArg) {
"arm-linux-androideabi" -> "armv7a-linux-androideabi"
else -> targetProperties.targetArg
}
private val prefix = "$absoluteTargetToolchain/bin/${clangQuad}${Android.API}"
private val clang = if (HostManager.hostIsMingw) "$prefix-clang.cmd" else "$prefix-clang"
private val ar = "$absoluteTargetToolchain/${targetProperties.targetArg}/bin/ar"
override val useCompilerDriverAsLinker: Boolean get() = true
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
val toolchainSysroot = "${absoluteTargetToolchain}/sysroot"
val architectureDir = Android.architectureDirForTarget(target)
val apiSysroot = "$absoluteTargetSysRoot/$architectureDir"
val clangTarget = targetArg!!
val libDirs = listOf(
"--sysroot=$apiSysroot",
if (target == KonanTarget.ANDROID_X64) "-L$apiSysroot/usr/lib64" else "-L$apiSysroot/usr/lib",
"-L$toolchainSysroot/usr/lib/$clangTarget/${Android.API}",
"-L$toolchainSysroot/usr/lib/$clangTarget")
return listOf(Command(clang).apply {
+"-o"
+executable
+"-fPIC"
+"-shared"
+"-target"
+targetArg!!
+libDirs
+objectFiles
if (optimize) +linkerOptimizationFlags
if (!debug) +linkerNoDebugFlags
if (dynamic) +linkerDynamicFlags
if (dynamic) +"-Wl,-soname,${File(executable).name}"
+linkerKonanFlags
if (mimallocEnabled) +mimallocLinkerDependencies
+libraries
+linkerArgs
})
}
}
class MacOSBasedLinker(targetProperties: AppleConfigurables)
: LinkerFlags(targetProperties), AppleConfigurables by targetProperties {
private val libtool = "$absoluteTargetToolchain/usr/bin/libtool"
private val linker = "$absoluteTargetToolchain/usr/bin/ld"
private val strip = "$absoluteTargetToolchain/usr/bin/strip"
private val dsymutil = "$absoluteTargetToolchain/usr/bin/dsymutil"
private val KonanTarget.isSimulator: Boolean
get() = this == KonanTarget.TVOS_X64 || this == KonanTarget.IOS_X64 ||
this == KonanTarget.WATCHOS_X86 || this == KonanTarget.WATCHOS_X64
override fun provideCompilerRtLibrary(libraryName: String): String? {
val prefix = when (target.family) {
Family.IOS -> "ios"
Family.WATCHOS -> "watchos"
Family.TVOS -> "tvos"
Family.OSX -> "osx"
else -> error("Target $target is unsupported")
}
val suffix = if (libraryName.isNotEmpty() && target.isSimulator) {
"sim"
} else {
""
}
val dir = File("$absoluteTargetToolchain/usr/lib/clang/").listFiles.firstOrNull()?.absolutePath
val mangledLibraryName = if (libraryName.isEmpty()) "" else "${libraryName}_"
return if (dir != null) "$dir/lib/darwin/libclang_rt.$mangledLibraryName$prefix$suffix.a" else null
}
private val osVersionMinFlags: List<String> by lazy {
listOf(osVersionMinFlagLd, osVersionMin + ".0")
}
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
// Note that may break in case of 32-bit Mach-O. See KT-37368.
override fun preLinkCommands(objectFiles: List<ObjectFile>, output: ObjectFile): List<Command> =
Command(linker).apply {
+"-r"
+listOf("-arch", arch)
+listOf("-syslibroot", absoluteTargetSysRoot)
+objectFiles
+listOf("-o", output)
}.let(::listOf)
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean, kind: LinkerOutputKind,
outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return listOf(Command(libtool).apply {
+"-static"
+listOf("-o", executable)
+objectFiles
+libraries
})
val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
val result = mutableListOf<Command>()
result += Command(linker).apply {
+"-demangle"
+listOf("-dynamic", "-arch", arch)
+osVersionMinFlags
+listOf("-syslibroot", absoluteTargetSysRoot, "-o", executable)
+objectFiles
if (optimize) +linkerOptimizationFlags
if (!debug) +linkerNoDebugFlags
if (dynamic) +linkerDynamicFlags
+linkerKonanFlags
if (mimallocEnabled) +mimallocLinkerDependencies
if (compilerRtLibrary != null) +compilerRtLibrary!!
if (needsProfileLibrary) +profileLibrary!!
+libraries
+linkerArgs
+rpath(dynamic)
}
// TODO: revise debug information handling.
if (debug) {
result += dsymUtilCommand(executable, outputDsymBundle)
if (optimize) {
result += Command(strip, "-S", executable)
}
}
return result
}
private val compilerRtLibrary: String? by lazy {
provideCompilerRtLibrary("")
}
private fun rpath(dynamic: Boolean): List<String> = listOfNotNull(
when (target.family) {
Family.OSX -> "@executable_path/../Frameworks"
Family.IOS,
Family.WATCHOS,
Family.TVOS -> "@executable_path/Frameworks"
else -> error(target)
},
"@loader_path/Frameworks".takeIf { dynamic }
).flatMap { listOf("-rpath", it) }
fun dsymUtilCommand(executable: ExecutableFile, outputDsymBundle: String) =
object : Command(dsymutilCommand(executable, outputDsymBundle)) {
override fun runProcess(): Int =
executeCommandWithFilter(command)
}
// TODO: consider introducing a better filtering directly in Command.
private fun executeCommandWithFilter(command: List<String>): Int {
val builder = ProcessBuilder(command)
// Inherit main process output streams.
val isDsymUtil = (command[0] == dsymutil)
builder.redirectOutput(Redirect.INHERIT)
builder.redirectInput(Redirect.INHERIT)
if (!isDsymUtil)
builder.redirectError(Redirect.INHERIT)
val process = builder.start()
if (isDsymUtil) {
/**
* llvm-lto has option -alias that lets tool to know which symbol we use instead of _main,
* llvm-dsym doesn't have such a option, so we ignore annoying warning manually.
*/
val errorStream = process.errorStream
val outputStream = bufferedReader(errorStream)
while (true) {
val line = outputStream.readLine() ?: break
if (!line.contains("warning: could not find object file symbol for symbol _main"))
System.err.println(line)
}
outputStream.close()
}
val exitCode = process.waitFor()
return exitCode
}
fun dsymutilCommand(executable: ExecutableFile, outputDsymBundle: String): List<String> =
listOf(dsymutil, executable, "-o", outputDsymBundle)
fun dsymutilDryRunVerboseCommand(executable: ExecutableFile): List<String> =
listOf(dsymutil, "-dump-debug-map", executable)
}
class GccBasedLinker(targetProperties: GccConfigurables)
: LinkerFlags(targetProperties), GccConfigurables by targetProperties {
private val ar = if (HostManager.hostIsMac) "$absoluteTargetToolchain/bin/llvm-ar"
else "$absoluteTargetToolchain/bin/ar"
override val libGcc = "$absoluteTargetSysRoot/${super.libGcc}"
private val linker = "$absoluteLlvmHome/bin/ld.lld"
private val specificLibs = abiSpecificLibraries.map { "-L${absoluteTargetSysRoot}/$it" }
override fun provideCompilerRtLibrary(libraryName: String): String? {
val targetSuffix = when (target) {
KonanTarget.LINUX_X64 -> "x86_64"
else -> error("$target is not supported.")
}
val dir = File("$absoluteLlvmHome/lib/clang/").listFiles.firstOrNull()?.absolutePath
return if (dir != null) "$dir/lib/linux/libclang_rt.$libraryName-$targetSuffix.a" else null
}
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isUnixStaticLib }
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
val isMips = target == KonanTarget.LINUX_MIPS32 || target == KonanTarget.LINUX_MIPSEL32
val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
val crtPrefix = if (configurables.target == KonanTarget.LINUX_ARM64) "usr/lib" else "usr/lib64"
// TODO: Can we extract more to the konan.configurables?
return listOf(Command(linker).apply {
+"--sysroot=${absoluteTargetSysRoot}"
+"-export-dynamic"
+"-z"
+"relro"
+"--build-id"
+"--eh-frame-hdr"
+"-dynamic-linker"
+dynamicLinker
+"-o"
+executable
if (!dynamic) +"$absoluteTargetSysRoot/$crtPrefix/crt1.o"
+"$absoluteTargetSysRoot/$crtPrefix/crti.o"
+if (dynamic) "$libGcc/crtbeginS.o" else "$libGcc/crtbegin.o"
+"-L$llvmLib"
+"-L$libGcc"
if (!isMips) +"--hash-style=gnu" // MIPS doesn't support hash-style=gnu
+specificLibs
+listOf("-L$absoluteTargetSysRoot/../lib", "-L$absoluteTargetSysRoot/lib", "-L$absoluteTargetSysRoot/usr/lib")
if (optimize) +linkerOptimizationFlags
if (!debug) +linkerNoDebugFlags
if (dynamic) +linkerDynamicFlags
+objectFiles
// See explanation about `-u__llvm_profile_runtime` here:
// https://github.com/llvm/llvm-project/blob/21e270a479a24738d641e641115bce6af6ed360a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L930
if (needsProfileLibrary) +listOf("-u__llvm_profile_runtime", profileLibrary!!)
+linkerKonanFlags
+listOf("-lgcc", "--as-needed", "-lgcc_s", "--no-as-needed",
"-lc", "-lgcc", "--as-needed", "-lgcc_s", "--no-as-needed")
+if (dynamic) "$libGcc/crtendS.o" else "$libGcc/crtend.o"
+"$absoluteTargetSysRoot/$crtPrefix/crtn.o"
+libraries
+linkerArgs
if (mimallocEnabled) +mimallocLinkerDependencies
})
}
}
class MingwLinker(targetProperties: MingwConfigurables)
: LinkerFlags(targetProperties), MingwConfigurables by targetProperties {
private val ar = "$absoluteTargetToolchain/bin/ar"
private val linker = "$absoluteTargetToolchain/bin/clang++"
override val useCompilerDriverAsLinker: Boolean get() = true
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isWindowsStaticLib || it.isUnixStaticLib }
override fun provideCompilerRtLibrary(libraryName: String): String? {
val targetSuffix = when (target) {
KonanTarget.MINGW_X64 -> "x86_64"
else -> error("$target is not supported.")
}
val dir = File("$absoluteLlvmHome/lib/clang/").listFiles.firstOrNull()?.absolutePath
return if (dir != null) "$dir/lib/windows/libclang_rt.$libraryName-$targetSuffix.a" else null
}
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind == LinkerOutputKind.STATIC_LIBRARY)
return staticGnuArCommands(ar, executable, objectFiles, libraries)
val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
return listOf(when {
HostManager.hostIsMingw -> Command(linker)
else -> Command("wine64", "$linker.exe")
}.apply {
+listOf("-o", executable)
+objectFiles
// --gc-sections flag may affect profiling.
// See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#drawbacks-and-limitations.
// TODO: switching to lld may help.
if (optimize && !needsProfileLibrary) +linkerOptimizationFlags
if (!debug) +linkerNoDebugFlags
if (dynamic) +linkerDynamicFlags
+libraries
if (needsProfileLibrary) +profileLibrary!!
+linkerArgs
+linkerKonanFlags
if (mimallocEnabled) +mimallocLinkerDependencies
})
}
}
class WasmLinker(targetProperties: WasmConfigurables)
: LinkerFlags(targetProperties), WasmConfigurables by targetProperties {
override val useCompilerDriverAsLinker: Boolean get() = false
override fun filterStaticLibraries(binaries: List<String>) = binaries.filter { it.isJavaScript }
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind")
val linkage = Command("$llvmBin/wasm-ld").apply {
+objectFiles
+listOf("-o", executable)
+lldFlags
}
// TODO(horsh): maybe rethink it.
val jsBindingsGeneration = object : Command() {
override fun execute() {
javaScriptLink(libraries, executable)
}
private fun javaScriptLink(jsFiles: List<String>, executable: String): String {
val linkedJavaScript = File("$executable.js")
val linkerHeader = "var konan = { libraries: [] };\n"
val linkerFooter = """|if (isBrowser()) {
| konan.moduleEntry([]);
|} else {
| konan.moduleEntry(arguments);
|}""".trimMargin()
linkedJavaScript.writeText(linkerHeader)
jsFiles.forEach {
linkedJavaScript.appendBytes(File(it).readBytes())
}
linkedJavaScript.appendBytes(linkerFooter.toByteArray())
return linkedJavaScript.name
}
}
return listOf(linkage, jsBindingsGeneration)
}
}
open class ZephyrLinker(targetProperties: ZephyrConfigurables)
: LinkerFlags(targetProperties), ZephyrConfigurables by targetProperties {
private val linker = "$absoluteTargetToolchain/bin/ld"
override val useCompilerDriverAsLinker: Boolean get() = false
override fun filterStaticLibraries(binaries: List<String>) = emptyList<String>()
override fun finalLinkCommands(objectFiles: List<ObjectFile>, executable: ExecutableFile,
libraries: List<String>, linkerArgs: List<String>,
optimize: Boolean, debug: Boolean,
kind: LinkerOutputKind, outputDsymBundle: String,
needsProfileLibrary: Boolean, mimallocEnabled: Boolean): List<Command> {
if (kind != LinkerOutputKind.EXECUTABLE) throw Error("Unsupported linker output kind: $kind")
return listOf(Command(linker).apply {
+listOf("-r", "--gc-sections", "--entry", "main")
+listOf("-o", executable)
+objectFiles
+libraries
+linkerArgs
})
}
}
fun linker(configurables: Configurables): LinkerFlags =
when (configurables) {
is GccConfigurables -> GccBasedLinker(configurables)
is AppleConfigurables -> MacOSBasedLinker(configurables)
is AndroidConfigurables-> AndroidLinker(configurables)
is MingwConfigurables -> MingwLinker(configurables)
is WasmConfigurables -> WasmLinker(configurables)
is ZephyrConfigurables -> ZephyrLinker(configurables)
else -> error("Unexpected target: ${configurables.target}")
}
@@ -0,0 +1,56 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.util.DependencyProcessor
class Platform(val configurables: Configurables)
: Configurables by configurables {
val clang by lazy {
ClangArgs(configurables)
}
val linker by lazy {
linker(configurables)
}
}
class PlatformManager(private val distribution: Distribution, experimental: Boolean = false) :
HostManager(distribution, experimental) {
constructor(konanHome: String, experimental: Boolean = false): this(Distribution(konanHome), experimental)
private val loaders = filteredOutEnabledButNotSupported.map {
it to loadConfigurables(it, distribution.properties, DependencyProcessor.defaultDependenciesRoot.absolutePath)
}.toMap()
private val platforms = loaders.map {
it.key to Platform(it.value)
}.toMap()
fun platform(target: KonanTarget) = platforms.getValue(target)
val hostPlatform = platforms.getValue(host)
fun loader(target: KonanTarget) = loaders.getValue(target)
/**
* TODO: Don't forget to delete this field and replace all its usages to `enabled`.
*/
val filteredOutEnabledButNotSupported
get() = enabled.filterNot { it == KonanTarget.WATCHOS_X64 }
}
@@ -0,0 +1,37 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.properties
import org.jetbrains.kotlin.konan.target.*
fun Properties.hostString(name: String, host: KonanTarget): String?
= this.resolvablePropertyString(name, host.name)
fun Properties.hostList(name: String, host: KonanTarget): List<String>
= this.resolvablePropertyList(name, host.name)
fun Properties.targetString(name: String, target: KonanTarget): String?
= this.resolvablePropertyString(name, target.name)
fun Properties.targetList(name: String, target: KonanTarget): List<String>
= this.resolvablePropertyList(name, target.name)
fun Properties.hostTargetString(name: String, target: KonanTarget, host: KonanTarget): String?
= this.resolvablePropertyString(name, hostTargetSuffix(host, target))
fun Properties.hostTargetList(name: String, target: KonanTarget, host: KonanTarget): List<String>
= this.resolvablePropertyList(name, hostTargetSuffix(host, target))
@@ -0,0 +1,81 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.target
import org.jetbrains.kotlin.konan.KonanExternalToolFailure
import org.jetbrains.kotlin.konan.MissingXcodeException
import org.jetbrains.kotlin.konan.exec.Command
import org.jetbrains.kotlin.konan.file.File
interface Xcode {
val toolchain: String
val macosxSdk: String
val iphoneosSdk: String
val iphonesimulatorSdk: String
val version: String
val appletvosSdk: String
val appletvsimulatorSdk: String
val watchosSdk: String
val watchsimulatorSdk: String
// Xcode.app/Contents/Developer/usr
val additionalTools: String
val simulatorRuntimes: String
companion object {
val current: Xcode by lazy {
CurrentXcode
}
}
}
private object CurrentXcode : Xcode {
override val toolchain by lazy {
val ldPath = xcrun("-f", "ld") // = $toolchain/usr/bin/ld
File(ldPath).parentFile.parentFile.parentFile.absolutePath
}
override val additionalTools: String by lazy {
val bitcodeBuildToolPath = xcrun("-f", "bitcode-build-tool")
File(bitcodeBuildToolPath).parentFile.parentFile.absolutePath
}
override val simulatorRuntimes: String by lazy {
Command("/usr/bin/xcrun", "simctl", "list", "runtimes", "-j").getOutputLines().joinToString(separator = "\n")
}
override val macosxSdk by lazy { getSdkPath("macosx") }
override val iphoneosSdk by lazy { getSdkPath("iphoneos") }
override val iphonesimulatorSdk by lazy { getSdkPath("iphonesimulator") }
override val appletvosSdk by lazy { getSdkPath("appletvos") }
override val appletvsimulatorSdk by lazy { getSdkPath("appletvsimulator") }
override val watchosSdk: String by lazy { getSdkPath("watchos") }
override val watchsimulatorSdk: String by lazy { getSdkPath("watchsimulator") }
override val version by lazy {
xcrun("xcodebuild", "-version")
.removePrefix("Xcode ")
}
private fun xcrun(vararg args: String): String = try {
Command("/usr/bin/xcrun", *args).getOutputLines().first()
} catch(e: KonanExternalToolFailure) {
throw MissingXcodeException("An error occurred during an xcrun execution. Make sure that Xcode and its command line tools are properly installed.", e)
}
private fun getSdkPath(sdk: String) = xcrun("--sdk", sdk, "--show-sdk-path")
}
@@ -0,0 +1,174 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
import java.io.File
import java.io.StringReader
import java.util.*
class DefFile(val file:File?, val config:DefFileConfig, val manifestAddendProperties:Properties, val defHeaderLines:List<String>) {
private constructor(file0:File?, triple: Triple<Properties, Properties, List<String>>): this(file0, DefFileConfig(triple.first), triple.second, triple.third)
constructor(file:File?, substitutions: Map<String, String>) : this(file, parseDefFile(file, substitutions))
val name by lazy {
file?.nameWithoutExtension ?: ""
}
class DefFileConfig(private val properties: Properties) {
val headers by lazy {
properties.getSpaceSeparated("headers")
}
val modules by lazy {
properties.getSpaceSeparated("modules")
}
val language by lazy {
properties.getProperty("language")
}
val compilerOpts by lazy {
properties.getSpaceSeparated("compilerOpts")
}
val excludeSystemLibs by lazy {
properties.getProperty("excludeSystemLibs")?.toBoolean() ?: false
}
val excludeDependentModules by lazy {
properties.getProperty("excludeDependentModules")?.toBoolean() ?: false
}
val entryPoints by lazy {
properties.getSpaceSeparated("entryPoint")
}
val linkerOpts by lazy {
properties.getSpaceSeparated("linkerOpts")
}
val linker by lazy {
properties.getProperty("linker", "clang")
}
val excludedFunctions by lazy {
properties.getSpaceSeparated("excludedFunctions")
}
val excludedMacros by lazy {
properties.getSpaceSeparated("excludedMacros")
}
val staticLibraries by lazy {
properties.getSpaceSeparated("staticLibraries")
}
val libraryPaths by lazy {
properties.getSpaceSeparated("libraryPaths")
}
val packageName by lazy {
properties.getProperty("package")
}
val headerFilter by lazy {
properties.getSpaceSeparated("headerFilter")
}
val strictEnums by lazy {
properties.getSpaceSeparated("strictEnums")
}
val nonStrictEnums by lazy {
properties.getSpaceSeparated("nonStrictEnums")
}
val noStringConversion by lazy {
properties.getSpaceSeparated("noStringConversion")
}
val depends by lazy {
properties.getSpaceSeparated("depends")
}
val exportForwardDeclarations by lazy {
properties.getSpaceSeparated("exportForwardDeclarations")
}
val disableDesignatedInitializerChecks by lazy {
properties.getProperty("disableDesignatedInitializerChecks")?.toBoolean() ?: false
}
val foreignExceptionMode by lazy {
properties.getProperty("foreignExceptionMode")
}
}
}
private fun Properties.getSpaceSeparated(name: String): List<String> =
this.getProperty(name)?.let { parseSpaceSeparatedArgs(it) } ?: emptyList()
private fun parseDefFile(file: File?, substitutions: Map<String, String>): Triple<Properties, Properties, List<String>> {
val properties = Properties()
if (file == null) {
return Triple(properties, Properties(), emptyList())
}
val lines = file.readLines()
val separator = "---"
val separatorIndex = lines.indexOf(separator)
val propertyLines: List<String>
val headerLines: List<String>
if (separatorIndex != -1) {
propertyLines = lines.subList(0, separatorIndex)
headerLines = lines.subList(separatorIndex + 1, lines.size)
} else {
propertyLines = lines
headerLines = emptyList()
}
// \ isn't escaping character in quotes, so replace them with \\.
val joinedLines = propertyLines.joinToString(System.lineSeparator())
val escapedTokens = joinedLines.split('"')
val postprocessProperties = escapedTokens.mapIndexed { index, token ->
if (index % 2 != 0) {
token.replace("""\\(?=.)""".toRegex(), Regex.escapeReplacement("""\\"""))
} else {
token
}
}.joinToString("\"")
val propertiesReader = StringReader(postprocessProperties)
properties.load(propertiesReader)
// Pass unsubstituted copy of properties we have obtained from `.def`
// to compiler `-manifest`.
val manifestAddendProperties = properties.duplicate()
substitute(properties, substitutions)
return Triple(properties, manifestAddendProperties, headerLines)
}
private fun Properties.duplicate() = Properties().apply { putAll(this@duplicate) }
fun DefFile(file: File?, target: KonanTarget) = DefFile(file, defaultTargetSubstitutions(target))
@@ -0,0 +1,230 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.util
import java.io.*
import java.net.HttpURLConnection
import java.net.URL
import java.net.URLConnection
import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.*
typealias ProgressCallback = (url: String, currentBytes: Long, totalBytes: Long) -> Unit
class DependencyDownloader(
var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS,
var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS,
customProgressCallback: ProgressCallback? = null
) {
private val progressCallback = customProgressCallback ?: { url, currentBytes, totalBytes ->
print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ")
}
val executor = ExecutorCompletionService<Unit>(Executors.newSingleThreadExecutor(object : ThreadFactory {
override fun newThread(r: Runnable?): Thread {
val thread = Thread(r)
thread.name = "konan-dependency-downloader"
thread.isDaemon = true
return thread
}
}))
enum class ReplacingMode {
/** Redownload the file and replace the existing one. */
REPLACE,
/** Throw FileAlreadyExistsException */
THROW,
/** Don't download the file and return the existing one*/
RETURN_EXISTING
}
class HTTPResponseException(val url: URL, val responseCode: Int)
: IOException("Server returned HTTP response code: $responseCode for URL: $url")
class DownloadingProgress(@Volatile var currentBytes: Long) {
fun update(readBytes: Int) { currentBytes += readBytes }
}
private fun HttpURLConnection.checkHTTPResponse(expected: Int, originalUrl: URL = url) {
if (responseCode != expected) {
throw HTTPResponseException(originalUrl, responseCode)
}
}
private fun HttpURLConnection.checkHTTPResponse(originalUrl: URL, predicate: (Int) -> Boolean) {
if (!predicate(responseCode)) {
throw HTTPResponseException(originalUrl, responseCode)
}
}
private fun doDownload(originalUrl: URL,
connection: URLConnection,
tmpFile: File,
currentBytes: Long,
totalBytes: Long,
append: Boolean) {
val progress = DownloadingProgress(currentBytes)
// TODO: Implement multi-thread downloading.
executor.submit {
connection.getInputStream().use { from ->
FileOutputStream(tmpFile, append).use { to ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var read = from.read(buffer)
while (read != -1) {
if (Thread.interrupted()) {
throw InterruptedException()
}
to.write(buffer, 0, read)
progress.update(read)
read = from.read(buffer)
}
if (progress.currentBytes != totalBytes) {
throw EOFException("The stream closed before end of downloading.")
}
}
}
}
var result: Future<Unit>?
do {
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
result = executor.poll(1, TimeUnit.SECONDS)
} while(result == null)
progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes)
try {
result.get()
} catch (e: ExecutionException) {
throw e.cause ?: e
}
}
private fun resumeDownload(originalUrl: URL, originalConnection: HttpURLConnection, tmpFile: File) {
originalConnection.connect()
val totalBytes = originalConnection.contentLengthLong
val currentBytes = tmpFile.length()
if (currentBytes >= totalBytes || originalConnection.getHeaderField("Accept-Ranges") != "bytes") {
// The temporary file is bigger then expected or the server doesn't support resuming downloading.
// Download the file from scratch.
doDownload(originalUrl, originalConnection, tmpFile, 0, totalBytes, false)
} else {
originalConnection.disconnect()
val rangeConnection = originalUrl.openConnection() as HttpURLConnection
rangeConnection.setRequestProperty("range", "bytes=$currentBytes-")
rangeConnection.connect()
rangeConnection.checkHTTPResponse(originalUrl) {
it == HttpURLConnection.HTTP_PARTIAL || it == HttpURLConnection.HTTP_OK
}
doDownload(originalUrl, rangeConnection, tmpFile, currentBytes, totalBytes, true)
}
}
/** Performs an attempt to download a specified file into the specified location */
private fun tryDownload(url: URL, tmpFile: File) {
val connection = url.openConnection()
(connection as? HttpURLConnection)?.checkHTTPResponse(HttpURLConnection.HTTP_OK, url)
if (connection is HttpURLConnection && tmpFile.exists()) {
resumeDownload(url, connection, tmpFile)
} else {
connection.connect()
val totalBytes = connection.contentLengthLong
doDownload(url, connection, tmpFile, 0, totalBytes, false)
}
}
/** Downloads a file from [source] url to [destination]. Returns [destination]. */
fun download(source: URL,
destination: File,
replace: ReplacingMode = ReplacingMode.RETURN_EXISTING): File {
if (destination.exists()) {
when (replace) {
ReplacingMode.RETURN_EXISTING -> return destination
ReplacingMode.THROW -> throw FileAlreadyExistsException(destination)
ReplacingMode.REPLACE -> Unit // Just continue with downloading.
}
}
val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX")
check(!tmpFile.isDirectory) {
"A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again."
}
check(!destination.isDirectory) {
"The destination file is a directory: ${destination.canonicalPath}. Remove it and try again."
}
var attempt = 1
var waitTime = 0L
val handleException = { e: Exception ->
if (attempt >= maxAttempts) {
throw e
}
attempt++
waitTime += attemptIntervalMs
println("Cannot download a dependency: $e\n" +
"Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts).")
// TODO: Wait better
Thread.sleep(waitTime)
}
while (true) {
try {
tryDownload(source, tmpFile)
break
} catch (e: HTTPResponseException) {
if (e.responseCode >= 500) {
// Retry server errors.
handleException(e)
} else {
// Do not retry client errors.
throw e
}
} catch (e: IOException) {
handleException(e)
}
}
Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING)
println("Done.")
return destination
}
private val Long.humanReadable: String
get() {
if (this < 0) {
return "-"
}
if (this < 1024) {
return "$this bytes"
}
val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt()
val prefix = "kMGTPE"[exp-1]
return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix)
}
companion object {
const val DEFAULT_MAX_ATTEMPTS = 10
const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L
const val TMP_SUFFIX = "part"
}
}
@@ -0,0 +1,73 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.unzipTo
import java.io.File
import java.util.concurrent.TimeUnit
enum class ArchiveType(val fileExtension: String) {
ZIP("zip"),
TAR_GZ("tar.gz");
companion object {
val systemDefault = if (System.getProperty("os.name").startsWith("Windows")) {
ZIP
} else {
TAR_GZ
}
}
}
class DependencyExtractor(
private val archiveType: ArchiveType
) {
private fun extractTarGz(tarGz: File, targetDirectory: File) {
val tarProcess = ProcessBuilder().apply {
command("tar", "-xzf", tarGz.canonicalPath)
directory(targetDirectory)
inheritIO()
}.start()
val finished = tarProcess.waitFor(extractionTimeout, extractionTimeoutUntis)
when {
finished && tarProcess.exitValue() != 0 ->
throw RuntimeException(
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
"Tar exit code: ${tarProcess.exitValue()}."
)
!finished -> {
tarProcess.destroy()
throw RuntimeException(
"Cannot extract archive with dependency: ${tarGz.canonicalPath}.\n" +
"Tar process hasn't finished in ${extractionTimeoutUntis.toSeconds(extractionTimeout)} sec.")
}
}
}
fun extract(archive: File, targetDirectory: File) {
when (archiveType) {
ArchiveType.ZIP -> archive.toPath().unzipTo(targetDirectory.toPath())
ArchiveType.TAR_GZ -> extractTarGz(archive, targetDirectory)
}
}
companion object {
val extractionTimeout = 3600L
val extractionTimeoutUntis = TimeUnit.SECONDS
}
}
@@ -0,0 +1,335 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.file.use
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.propertyList
import java.io.File
import java.io.FileNotFoundException
import java.io.RandomAccessFile
import java.net.InetAddress
import java.net.URL
import java.net.UnknownHostException
import java.nio.file.Paths
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
private val Properties.dependenciesUrl : String
get() = getProperty("dependenciesUrl")
?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl")
private val Properties.airplaneMode : Boolean
get() = getProperty("airplaneMode")?.toBoolean() ?: false
private val Properties.downloadingAttempts : Int
get() = getProperty("downloadingAttempts")?.toInt()
?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS
private val Properties.downloadingAttemptIntervalMs : Long
get() = getProperty("downloadingAttemptPauseMs")?.toLong()
?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS
private fun Properties.findCandidates(dependencies: List<String>): Map<String, List<DependencySource>> {
val dependencyProfiles = this.propertyList("dependencyProfiles")
return dependencies.map { dependency ->
dependency to dependencyProfiles.flatMap { profile ->
val candidateSpecs = propertyList("$dependency.$profile")
if (profile == "default" && candidateSpecs.isEmpty()) {
listOf(DependencySource.Remote.Public)
} else {
candidateSpecs.map { candidateSpec ->
when (candidateSpec) {
"remote:public" -> DependencySource.Remote.Public
"remote:internal" -> DependencySource.Remote.Internal
else -> DependencySource.Local(File(candidateSpec))
}
}
}
}
}.toMap()
}
private val KonanPropertiesLoader.dependenciesUrl : String get() = properties.dependenciesUrl
private val KonanPropertiesLoader.airplaneMode : Boolean get() = properties.airplaneMode
private val KonanPropertiesLoader.downloadingAttempts : Int get() = properties.downloadingAttempts
private val KonanPropertiesLoader.downloadingAttemptIntervalMs : Long get() = properties.downloadingAttemptIntervalMs
sealed class DependencySource {
data class Local(val path: File) : DependencySource()
sealed class Remote : DependencySource() {
object Public : Remote()
object Internal : Remote()
}
}
/**
* Inspects [dependencies] and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl].
* If [airplaneMode] is true will throw a RuntimeException instead of downloading.
*/
class DependencyProcessor(dependenciesRoot: File,
private val dependenciesUrl: String,
dependencyToCandidates: Map<String, List<DependencySource>>,
homeDependencyCache: File = defaultDependencyCacheDir,
private val airplaneMode: Boolean = false,
maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS,
attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS,
customProgressCallback: ProgressCallback? = null,
private val keepUnstable: Boolean = true,
private val deleteArchives: Boolean = true,
private val archiveType: ArchiveType = ArchiveType.systemDefault) {
private val dependenciesDirectory = dependenciesRoot.apply { mkdirs() }
private val cacheDirectory = homeDependencyCache.apply { mkdirs() }
private val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() }
var showInfo = true
private var isInfoShown = false
private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback)
private val extractor = DependencyExtractor(archiveType)
constructor(dependenciesRoot: File,
properties: KonanPropertiesLoader,
dependenciesUrl: String = properties.dependenciesUrl,
keepUnstable:Boolean = true,
archiveType: ArchiveType = ArchiveType.systemDefault) : this(
dependenciesRoot,
properties.properties,
properties.dependencies,
dependenciesUrl,
keepUnstable = keepUnstable,
archiveType = archiveType)
constructor(dependenciesRoot: File,
properties: Properties,
dependencies: List<String>,
dependenciesUrl: String = properties.dependenciesUrl,
keepUnstable:Boolean = true,
archiveType: ArchiveType = ArchiveType.systemDefault) : this(
dependenciesRoot,
dependenciesUrl,
dependencyToCandidates = properties.findCandidates(dependencies),
airplaneMode = properties.airplaneMode,
maxAttempts = properties.downloadingAttempts,
attemptIntervalMs = properties.downloadingAttemptIntervalMs,
keepUnstable = keepUnstable,
archiveType = archiveType)
class DependencyFile(directory: File, fileName: String) {
val file = File(directory, fileName).apply { createNewFile() }
private val dependencies = file.readLines().toMutableSet()
fun contains(dependency: String) = dependencies.contains(dependency)
fun add(dependency: String) = dependencies.add(dependency)
fun remove(dependency: String) = dependencies.remove(dependency)
fun removeAndSave(dependency: String) {
remove(dependency)
save()
}
fun addAndSave(dependency: String) {
add(dependency)
save()
}
fun save() {
val writer = file.writer()
writer.use {
dependencies.forEach {
writer.write(it)
writer.write("\n")
}
}
}
}
private fun downloadDependency(dependency: String, baseUrl: String) {
val depDir = File(dependenciesDirectory, dependency)
val depName = depDir.name
val fileName = "$depName.${archiveType.fileExtension}"
val archive = cacheDirectory.resolve(fileName)
val url = URL("$baseUrl/$fileName")
val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted")
if (extractedDependencies.contains(depName) &&
depDir.exists() &&
depDir.isDirectory &&
depDir.list().isNotEmpty()) {
if (!keepUnstable && depDir.list().contains(".unstable")) {
// The downloaded version of the dependency is unstable -> redownload it.
depDir.deleteRecursively()
archive.delete()
extractedDependencies.removeAndSave(dependency)
} else {
return
}
}
if (showInfo && !isInfoShown) {
println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.")
isInfoShown = true
}
if (!archive.exists()) {
if (airplaneMode) {
throw FileNotFoundException("""
Cannot find a dependency locally: $dependency.
Set `airplaneMode = false` in konan.properties to download it.
""".trimIndent())
}
downloader.download(url, archive)
}
println("Extracting dependency: $archive into $dependenciesDirectory")
extractor.extract(archive, dependenciesDirectory)
if (deleteArchives) {
archive.delete()
}
extractedDependencies.addAndSave(depName)
}
companion object {
val localKonanDir: File by lazy {
File(System.getenv("KONAN_DATA_DIR") ?: (System.getProperty("user.home") + File.separator + ".konan"))
}
@JvmStatic
val defaultDependenciesRoot: File
get() = localKonanDir.resolve("dependencies")
val defaultDependencyCacheDir: File
get() = localKonanDir.resolve("cache")
val isInternalSeverAvailable: Boolean
get() = InternalServer.isAvailable
}
private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) ->
val candidate = candidates.asSequence().mapNotNull { candidate ->
when (candidate) {
is DependencySource.Local -> candidate.takeIf { it.path.exists() }
DependencySource.Remote.Public -> candidate
DependencySource.Remote.Internal -> candidate.takeIf { InternalServer.isAvailable }
}
}.firstOrNull()
candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}")
dependency to candidate
}.toMap()
private fun resolveDependency(dependency: String): File {
val candidate = resolvedDependencies[dependency]
return when (candidate) {
is DependencySource.Local -> candidate.path
is DependencySource.Remote -> File(dependenciesDirectory, dependency)
null -> error("$dependency not declared as dependency")
}
}
/**
* If given [path] is relative, resolves it relative to dependecies directory.
* In case of absolute path just wraps it into a [File].
*
* Support of both relative and absolute path kinds allows to substitute predefined
* dependencies with system ones.
*
* TODO: It looks like DependencyProcessor have two split responsibilities:
* * Dependency resolving
* * Dependency downloading
* Also it is tightly tied to KonanProperties.
*/
fun resolve(path: String): File =
if (Paths.get(path).isAbsolute) File(path) else resolveRelative(path)
private fun resolveRelative(relative: String): File {
val path = Paths.get(relative)
if (path.isAbsolute) error("not a relative path: $relative")
val dependency = path.first().toString()
return resolveDependency(dependency).let {
if (path.nameCount > 1) {
it.toPath().resolve(path.subpath(1, path.nameCount)).toFile()
} else {
it
}
}
}
fun run() {
// We need a lock that can be shared between different classloaders (KT-39781).
// TODO: Rework dependencies downloading to avoid storing the lock in the system properties.
val lock = System.getProperties().computeIfAbsent("kotlin.native.dependencies.lock") {
// String literals are internalized so we create a new instance to avoid synchronization on a shared object.
java.lang.String("lock")
}
synchronized(lock) {
RandomAccessFile(lockFile, "rw").channel.lock().use {
resolvedDependencies.forEach { (dependency, candidate) ->
val baseUrl = when (candidate) {
is DependencySource.Local -> null
DependencySource.Remote.Public -> dependenciesUrl
DependencySource.Remote.Internal -> InternalServer.url
}
// TODO: consider using different caches for different remotes.
if (baseUrl != null) {
downloadDependency(dependency, baseUrl)
}
}
}
}
}
}
internal object InternalServer {
private const val host = "repo.labs.intellij.net"
const val url = "https://$host/kotlin-native"
private const val internalDomain = "labs.intellij.net"
val isAvailable: Boolean get() {
val envKey = "KONAN_USE_INTERNAL_SERVER"
return when (val envValue = System.getenv(envKey)) {
null, "0" -> false
"1" -> true
"auto" -> isAccessible
else -> error("unexpected environment: $envKey=$envValue")
}
}
private val isAccessible by lazy { checkAccessible() }
private fun checkAccessible() = try {
if (!InetAddress.getLocalHost().canonicalHostName.endsWith(".$internalDomain")) {
// Fast path:
false
} else {
InetAddress.getByName(host)
true
}
} catch (e: UnknownHostException) {
false
}
}
@@ -0,0 +1,5 @@
package org.jetbrains.kotlin.konan.util
object PlatformLibsInfo {
const val namePrefix = "org.jetbrains.kotlin.native.platform."
}
@@ -0,0 +1,29 @@
package org.jetbrains.kotlin.konan.util
import org.jetbrains.kotlin.konan.target.KonanTarget
import java.util.*
// FIXME(ddol): KLIB-REFACTORING-CLEANUP: remove the whole file!
fun defaultTargetSubstitutions(target: KonanTarget) =
mapOf<String, String>(
"target" to target.visibleName,
"arch" to target.architecture.visibleName,
"family" to target.family.visibleName)
// Performs substitution similar to:
// foo = ${foo} ${foo.${arch}} ${foo.${os}}
fun substitute(properties: Properties, substitutions: Map<String, String>) {
for (key in properties.stringPropertyNames()) {
for (substitution in substitutions.values) {
val suffix = ".$substitution"
if (key.endsWith(suffix)) {
val baseKey = key.removeSuffix(suffix)
val oldValue = properties.getProperty(baseKey, "")
val appendedValue = properties.getProperty(key, "")
val newValue = if (oldValue != "") "$oldValue $appendedValue" else appendedValue
properties.setProperty(baseKey, newValue)
}
}
}
}