diff --git a/backend.native/build.gradle b/backend.native/build.gradle index d67b359cb1c..97047f7f213 100644 --- a/backend.native/build.gradle +++ b/backend.native/build.gradle @@ -40,7 +40,6 @@ allprojects { } // TODO: probably we should use kotlin-compiler without bundled runtime } - } sourceSets { @@ -182,69 +181,47 @@ dependencies { build.dependsOn 'compilerClasses','cli_bcClasses','bc_frontendClasses' +// These files are built before the 'dist' is complete, +// so we provide custom values for +// -runtime, -properties, -library and -Djava.library.path + task stdlib(type: JavaExec) { - main 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath configurations.cli_bc - - args project(':runtime').file('src/main/kotlin') - - args '-output', project(':runtime').file('build/stdlib.kt.bc') - - args project.globalArgs - - jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs" - - dependsOn ':runtime:build' + main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' + classpath = project.configurations.cli_bc + jvmArgs "-ea", + "-Dkonan.home=${project.parent.file('dist')}", + "-Djava.library.path=${project.buildDir}/nativelibs" + args('-output', project(':runtime').file('build/stdlib.kt.bc'), + '-nolink', '-nostdlib', + '-runtime', project(':runtime').file('build/runtime.bc'), + '-properties', project(':backend.native').file('konan.properties'), + project(':runtime').file('src/main/kotlin'), + *project.globalArgs) outputs.file(project(':runtime').file('build/stdlib.kt.bc')) - - doFirst { - args '-runtime', project(':runtime').build.outputs.files.singleFile - } - environment 'LD_LIBRARY_PATH' : "${llvmDir}/lib:${project.buildDir}/nativelibs" + dependsOn ':runtime:build' } task start(type: JavaExec) { - main 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath configurations.cli_bc - - args project(':runtime').file('src/launcher/kotlin') - - args '-library', project(':runtime').file('build/stdlib.kt.bc') - - args '-output', project(':runtime').file('build/start.kt.bc') - - args project.globalArgs - - jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs" - - dependsOn 'stdlib' + main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' + classpath = project.configurations.cli_bc + jvmArgs "-ea", + "-Dkonan.home=${project.parent.file('dist')}", + "-Djava.library.path=${project.buildDir}/nativelibs" + args('-output', project(':runtime').file('build/start.kt.bc'), + '-nolink', '-nostdlib', + '-library', project(':runtime').file('build/stdlib.kt.bc'), + '-runtime', project(':runtime').file('build/runtime.bc'), + '-properties', project(':backend.native').file('konan.properties'), + project(':runtime').file('src/launcher/kotlin'), + *project.globalArgs) outputs.file(project(':runtime').file('build/start.kt.bc')) - - doFirst { - args '-runtime', project(':runtime').build.outputs.files.singleFile - } - environment 'LD_LIBRARY_PATH' : "${llvmDir}/lib:${project.buildDir}/nativelibs" + dependsOn ':runtime:build', 'stdlib' } -task run(type: JavaExec) { - main 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath configurations.cli_bc - - args 'tests/codegen/function/sum.kt' - - args '-library', project(':runtime').file('build/stdlib.kt.bc') - - args '-output', "$buildDir/out.bc" - - jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs" - - dependsOn ':runtime:build' - dependsOn 'stdlib' - - doFirst { - args '-runtime', project(':runtime').build.outputs.files.singleFile - } - environment 'LD_LIBRARY_PATH' : "${llvmDir}/lib:${project.buildDir}/nativelibs" +task run { + doLast { + logger.quiet("Run the outer project 'demo' target to compile the test source." ) + } } task jars(type: Jar) { diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index 67efccb3a68..fc159c75f6d 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -11,6 +11,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.JvmPackagePartProvider import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.backend.konan.KonanConfigKeys +import org.jetbrains.kotlin.backend.konan.Distribution import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.config.addKotlinSourceRoots @@ -56,31 +57,51 @@ class K2Native : CLICompiler() { configuration.addKotlinSourceRoots(arguments.freeArgs) - // This is a decision we could change - configuration.put(CommonConfigurationKeys.MODULE_NAME, arguments.outputFile) + with (KonanConfigKeys) { with (configuration) { - configuration.put(KonanConfigKeys.LIBRARY_FILES, - arguments.libraries.toNonNullList()) - configuration.put(KonanConfigKeys.RUNTIME_FILE, arguments.runtimeFile) - configuration.put(KonanConfigKeys.OUTPUT_FILE, arguments.outputFile) + put(NOSTDLIB, arguments.nostdlib) + put(NOLINK, arguments.nolink) + put(LIBRARY_FILES, + arguments.libraries.toNonNullList()) - configuration.put(KonanConfigKeys.ABI_VERSION, 1) + // TODO: Collect all the explicit file names into an object + // and teach the compiler to work with temporaries and -save-temps. + val bitcodeFile = if (arguments.nolink) { + arguments.outputFile ?: "program.kt.bc" + } else { + "${arguments.outputFile ?: "program"}.kt.bc" + } + put(BITCODE_FILE, bitcodeFile) - configuration.put(KonanConfigKeys.PRINT_IR, arguments.printIr) - configuration.put(KonanConfigKeys.PRINT_DESCRIPTORS, arguments.printDescriptors) - configuration.put(KonanConfigKeys.PRINT_BITCODE, arguments.printBitCode) + // This is a decision we could change + put(CommonConfigurationKeys.MODULE_NAME, bitcodeFile) + put(ABI_VERSION, 1) - configuration.put(KonanConfigKeys.VERIFY_IR, arguments.verifyIr) - configuration.put(KonanConfigKeys.VERIFY_DESCRIPTORS, arguments.verifyDescriptors) - configuration.put(KonanConfigKeys.VERIFY_BITCODE, arguments.verifyBitCode) + put(EXECUTABLE_FILE, + arguments.outputFile ?: "program.kexe") + put(RUNTIME_FILE, + arguments.runtimeFile ?: Distribution.runtime) + put(PROPERTY_FILE, + arguments.propertyFile ?: Distribution.propertyFile) - configuration.put(KonanConfigKeys.ENABLED_PHASES, - arguments.enablePhases.toNonNullList()) - configuration.put(KonanConfigKeys.DISABLED_PHASES, - arguments.disablePhases.toNonNullList()) - configuration.put(KonanConfigKeys.VERBOSE_PHASES, - arguments.verbosePhases.toNonNullList()) - configuration.put(KonanConfigKeys.LIST_PHASES, arguments.listPhases) + put(OPTIMIZATION, arguments.optimization) + + put(PRINT_IR, arguments.printIr) + put(PRINT_DESCRIPTORS, arguments.printDescriptors) + put(PRINT_BITCODE, arguments.printBitCode) + + put(VERIFY_IR, arguments.verifyIr) + put(VERIFY_DESCRIPTORS, arguments.verifyDescriptors) + put(VERIFY_BITCODE, arguments.verifyBitCode) + + put(ENABLED_PHASES, + arguments.enablePhases.toNonNullList()) + put(DISABLED_PHASES, + arguments.disablePhases.toNonNullList()) + put(VERBOSE_PHASES, + arguments.verbosePhases.toNonNullList()) + put(LIST_PHASES, arguments.listPhases) + }} } override fun createArguments(): K2NativeCompilerArguments { diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java index ffa36736273..6ca5193482e 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java @@ -5,18 +5,31 @@ import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments; import org.jetbrains.kotlin.cli.common.arguments.ValueDescription; public class K2NativeCompilerArguments extends CommonCompilerArguments { - @Argument(value = "output", description = "Output file path") + @Argument(value = "output", alias = "o", description = "Output file path") @ValueDescription("") public String outputFile; - @Argument(value = "runtime", description = "Runtime file path") + @Argument(value = "runtime", description = "Override standard \'runtime.bc\' location") @ValueDescription("") public String runtimeFile; - @Argument(value = "library", description = "Bitcode file with metadata attached") + @Argument(value = "properties", description = "Override standard \'konan.properties\' location") + @ValueDescription("") + public String propertyFile; + + @Argument(value = "library", alias = "l", description = "Link with the library") @ValueDescription("") public String[] libraries; + @Argument(value = "nolink", description = "Don't link, just produce a bitcode file") + public boolean nolink; + + @Argument(value = "nostdlib", description = "Don't link with stdlib") + public boolean nostdlib; + + @Argument(value = "opt", description = "Enable optimizations during compilation") + public boolean optimization; + @Argument(value = "print_ir", description = "Print IR") public boolean printIr; diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt new file mode 100644 index 00000000000..789bc0f7109 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt @@ -0,0 +1,48 @@ +package org.jetbrains.kotlin.backend.konan + +import java.io.File +import java.util.Properties + +public class Distribution(val properties: KonanProperties, + val os: String, val target: String) { + + companion object { + + private fun findKonanHome(): String { + val value = System.getProperty("konan.home", "dist") + val path = File(value).absolutePath + return path + } + + val konanHome = findKonanHome() + val propertyFile = "$konanHome/konan/konan.properties" + + val lib = "$konanHome/lib" + + // TODO: Pack all needed things to dist. + val dependencies = "$konanHome/../dependencies/all" + + val start = "$lib/start.kt.bc" + val stdlib = "$lib/stdlib.kt.bc" + val runtime = "$lib/runtime.bc" + val launcher = "$lib/launcher.bc" + } + + val llvmHome = "$dependencies/${properties.propertyString("llvmHome.$os")}" + val sysRoot = "$dependencies/${properties.propertyString("sysRoot.$os")}" + val libGcc = "$dependencies/${properties.propertyString("libGcc.$os")}" + + val llvmBin = "$llvmHome/bin" + val llvmLib = "$llvmHome/lib" + + val llvmOpt = "$llvmBin/opt" + val llvmLlc = "$llvmBin/llc" + val llvmLto = "$llvmBin/llvm-lto" + val llvmLink = "$llvmBin/llvm-link" + val libCppAbi = "$llvmLib/libc++abi.a" + val libLTO = when (os) { + "osx" -> "$llvmLib/libLTO.dylib" + "linux" -> "$llvmLib/libLTO.so" + else -> error("Don't know libLTO location for the platform.") + } +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 0dbcffd5e88..e345ea58195 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -2,6 +2,7 @@ package org.jetbrains.kotlin.backend.konan import com.intellij.openapi.project.Project import org.jetbrains.kotlin.backend.konan.KonanPlatform +import org.jetbrains.kotlin.backend.konan.Distribution import org.jetbrains.kotlin.backend.konan.llvm.loadMetadata import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration @@ -14,7 +15,14 @@ import java.io.File class KonanConfig(val project: Project, val configuration: CompilerConfiguration) { - private val libraries = configuration.getList(KonanConfigKeys.LIBRARY_FILES) + internal val libraries: List + get() { + val fromCommandLine = configuration.getList(KonanConfigKeys.LIBRARY_FILES) + if (configuration.get(KonanConfigKeys.NOSTDLIB) ?: false) { + return fromCommandLine + } + return fromCommandLine + Distribution.stdlib + } private val loadedDescriptors = loadLibMetadata(libraries) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index 784480ad96b..7e28278e7f9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -7,12 +7,22 @@ class KonanConfigKeys { companion object { val LIBRARY_FILES: CompilerConfigurationKey> = CompilerConfigurationKey.create("library file paths"); - val RUNTIME_FILE: CompilerConfigurationKey - = CompilerConfigurationKey.create("runtime file path"); - val OUTPUT_FILE: CompilerConfigurationKey - = CompilerConfigurationKey.create("output file path"); + val BITCODE_FILE: CompilerConfigurationKey + = CompilerConfigurationKey.create("emitted bitcode file path"); + val EXECUTABLE_FILE: CompilerConfigurationKey + = CompilerConfigurationKey.create("final executable file path"); + val RUNTIME_FILE: CompilerConfigurationKey + = CompilerConfigurationKey.create("override default runtime file path"); + val PROPERTY_FILE: CompilerConfigurationKey + = CompilerConfigurationKey.create("override default property file path"); val ABI_VERSION: CompilerConfigurationKey = CompilerConfigurationKey.create("current abi version"); + val OPTIMIZATION: CompilerConfigurationKey + = CompilerConfigurationKey.create("optimized compilation"); + val NOSTDLIB: CompilerConfigurationKey + = CompilerConfigurationKey.create("don't link with stdlib"); + val NOLINK: CompilerConfigurationKey + = CompilerConfigurationKey.create("don't link, only produce a bitcode file "); val SOURCE_MAP: CompilerConfigurationKey> = CompilerConfigurationKey.create("generate source map"); diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 7f537cc9988..ff83e454d96 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -71,8 +71,7 @@ public fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEn } phaser.phase(KonanPhase.LINKER) { - //TODO: We don't have it yet. - // invokeLinker() + LinkStage(context).linkStage() } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt index c6921446424..2c163eaf5ba 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt @@ -33,6 +33,10 @@ object KonanPhases { val verbose = config.configuration.get(KonanConfigKeys.VERBOSE_PHASES) verbose?.forEach { phases[it]!!.verbose = true } + + if (config.configuration.get(KonanConfigKeys.NOLINK) ?: false ) { + KonanPhase.LINKER.enabled = false + } } fun list() { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt new file mode 100644 index 00000000000..e8770ee8887 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt @@ -0,0 +1,25 @@ +package org.jetbrains.kotlin.backend.konan + +import org.jetbrains.kotlin.config.CompilerConfiguration +import java.io.File +import java.util.Properties + +public class KonanProperties(config: CompilerConfiguration) { + + val properties = Properties() + + init { + val file = File(config.get(KonanConfigKeys.PROPERTY_FILE)) + file.bufferedReader()?.use { reader -> + properties.load(reader) + } + } + + fun propertyString(key: String): String? = properties.getProperty(key) + + fun propertyList(key: String): List { + val value = properties.getProperty(key) + return value?.split(' ') ?: listOf() + } +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt new file mode 100644 index 00000000000..aa127e69e81 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt @@ -0,0 +1,203 @@ +package org.jetbrains.kotlin.backend.konan + +import java.io.File +import java.lang.ProcessBuilder +import java.lang.ProcessBuilder.Redirect + +typealias BitcodeFile = String +typealias ObjectFile = String +typealias ExecutableFile = String + +// Use "clang -v -save-temps" to write linkCommand() method +// for another implementation of this class. +internal abstract class PlatformFlags(val distrib: Distribution, + val properties: KonanProperties) { + + abstract val llvmLtoFlags: List + abstract val llvmLlcFlags: List + + abstract val linker: String + abstract val linkerKonanFlags: List + abstract val linkerOptimizationFlags: List + + abstract fun linkCommand(objectFiles: List, + executable: ExecutableFile, optimize: Boolean): List +} + + +internal class MacOSPlatform(distrib: Distribution, + properties: KonanProperties) : PlatformFlags(distrib, properties) { + + override val llvmLtoFlags = properties.propertyList("llvmLtoFlags.osx") + override val llvmLlcFlags = properties.propertyList("llvmLlcFlags.osx") + + override val linkerOptimizationFlags = + properties.propertyList("linkerOptimizationFlags.osx") + override val linkerKonanFlags = listOf(distrib.libCppAbi) + override val linker = "${distrib.sysRoot}/usr/bin/ld" + + override fun linkCommand(objectFiles: List, executable: String, optimize: Boolean): List { + + val sysRoot = distrib.sysRoot + val llvmLib = distrib.llvmLib + + return mutableListOf(linker, "-demangle") + + if (optimize) listOf("-object_path_lto", "temporary.o", "-lto_library", distrib.libLTO) else {listOf()} + + listOf( "-dynamic", "-arch", "x86_64", "-macosx_version_min") + + properties.propertyList("macosVersionMin.osx") + + listOf("-syslibroot", "$sysRoot", + "-o", executable) + + objectFiles + + if (optimize) linkerOptimizationFlags else {listOf()} + + linkerKonanFlags + + listOf("-lSystem", "$llvmLib/clang/3.8.0/lib/darwin/libclang_rt.osx.a") + } +} + +internal class LinuxPlatform(distrib: Distribution, + properties: KonanProperties) : PlatformFlags(distrib, properties) { + + override val llvmLtoFlags = properties.propertyList("llvmLtoFlags.linux") + override val llvmLlcFlags = properties.propertyList("llvmLlcFlags.linux") + + override val linkerOptimizationFlags = + properties.propertyList("linkerOptimizationFlags.linux") + override val linkerKonanFlags = properties.propertyList("linkerKonanFlags.linux") + override val linker = "${distrib.sysRoot}/../bin/ld.gold" + + val pluginOptimizationFlags = + properties.propertyList("pluginOptimizationFlags.linux") + + override fun linkCommand(objectFiles: List, executable: String, optimize: Boolean): List { + + val sysRoot = distrib.sysRoot + val llvmLib = distrib.llvmLib + val libGcc = distrib.libGcc + + // TODO: Can we extract more to the konan.properties? + return mutableListOf("$linker", + "--sysroot=${sysRoot}", + "-export-dynamic", "-z", "relro", "--hash-style=gnu", + "--build-id", "--eh-frame-hdr", "-m", "elf_x86_64", + "-dynamic-linker", "/lib64/ld-linux-x86-64.so.2", + "-o", executable, + "${sysRoot}/usr/lib64/crt1.o", "${sysRoot}/usr/lib64/crti.o", "${libGcc}/crtbegin.o", + "-L${llvmLib}", "-L${libGcc}", "-L${sysRoot}/../lib64", "-L${sysRoot}/lib64", + "-L${sysRoot}/usr/lib64", "-L${sysRoot}/../lib", "-L${sysRoot}/lib", "-L${sysRoot}/usr/lib") + + if (optimize) listOf("-plugin", "$llvmLib/LLVMgold.so") + pluginOptimizationFlags else {listOf()} + + objectFiles + + if (optimize) linkerOptimizationFlags else {listOf()} + + linkerKonanFlags + + listOf("-lgcc", "--as-needed", "-lgcc_s", "--no-as-needed", + "-lc", "-lgcc", "--as-needed", "-lgcc_s", "--no-as-needed", + "${libGcc}/crtend.o", + "${sysRoot}/usr/lib64/crtn.o") + } +} + + +internal class LinkStage(val context: Context) { + + private val javaOsName = System.getProperty("os.name") + private val javaArch = System.getProperty("os.arch") + + val os = when (javaOsName) { + "Mac OS X" -> "osx" + "Linux" -> "linux" + else -> error("Unknown operating system: ${javaOsName}") + } + val arch = when (javaArch) { + "x86_64" -> "x86_64" + "amd64" -> "x86_64" + else -> error("Unknown hardware platform: ${javaArch}") + } + + private val properties = KonanProperties(context.config.configuration) + private val distrib = Distribution(properties, os, arch) + + val platform: PlatformFlags = when (os) { + "linux" -> LinuxPlatform(distrib, properties) + "osx" -> MacOSPlatform(distrib, properties) + else -> error("Could not tell the current platform") + } + + val config = context.config.configuration + val optimize = config.get(KonanConfigKeys.OPTIMIZATION) ?: false + val emitted = config.get(KonanConfigKeys.BITCODE_FILE)!! + val nostdlib = config.get(KonanConfigKeys.NOSTDLIB) ?: false + val libraries = context.config.libraries + + fun llvmLto(files: List): ObjectFile { + // TODO: Make it a temporary file. + val combined = "combined.o" + + val tool = distrib.llvmLto + val command = mutableListOf(tool, "-o", combined) + if (optimize) { + command.addAll(platform.llvmLtoFlags) + } + command.addAll(files) + runTool(*command.toTypedArray()) + + return combined + } + + fun llvmLlc(file: BitcodeFile): ObjectFile { + // TODO: Make it a temporary file. + val objectFile = "$file.o" + + val tool = distrib.llvmLlc + val command = listOf(distrib.llvmLlc, "-o", objectFile, "-filetype=obj") + + properties.propertyList("llvmLlcFlags.$os") + listOf(file) + runTool(*command.toTypedArray()) + + return objectFile + } + + fun link(objectFiles: List): ExecutableFile { + val executable = config.get(KonanConfigKeys.EXECUTABLE_FILE)!! + + val linkCommand = platform.linkCommand(objectFiles, executable, optimize) + runTool(*linkCommand.toTypedArray()) + + return executable + } + + fun executeCommand(vararg command: String): Int { + + context.log("") + context.log(command.asList().joinToString(" ")) + + val builder = ProcessBuilder(command.asList()) + + // Inherit main process output streams. + builder.redirectOutput(Redirect.INHERIT) + builder.redirectInput(Redirect.INHERIT) + builder.redirectError(Redirect.INHERIT) + + val process = builder.start() + val exitCode = process.waitFor() + return exitCode + } + + fun runTool(vararg command: String) { + val code = executeCommand(*command) + if (code != 0) error("The ${command[0]} command returned non-zero exit code: $code.") + } + + fun linkStage() { + context.log("# Compiler root: ${Distribution.konanHome}") + + val bitcodeFiles = listOf(emitted, Distribution.start, Distribution.runtime, + Distribution.launcher) + libraries + + val objectFiles = if (optimize) { + listOf( llvmLto(bitcodeFiles ) ) + } else { + bitcodeFiles.map{ it -> llvmLlc(it) } + } + + val executable = link(objectFiles) + } +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index fdcc021bd1f..423ead8ec1d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -3,6 +3,7 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.* import llvm.* import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.backend.konan.Distribution import org.jetbrains.kotlin.backend.konan.hash.GlobalHash import org.jetbrains.kotlin.backend.konan.KonanConfigKeys import org.jetbrains.kotlin.backend.konan.descriptors.backingField diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 68e233f78a7..e04c9db17c6 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -49,7 +49,7 @@ internal fun emitLLVM(context: Context) { irModule.acceptVoid(MetadatorVisitor(context)) } - val outFile = context.config.configuration.get(KonanConfigKeys.OUTPUT_FILE)!! + val outFile = context.config.configuration.get(KonanConfigKeys.BITCODE_FILE)!! LLVMWriteBitcodeToFile(llvmModule, outFile) } diff --git a/backend.native/konan.properties b/backend.native/konan.properties new file mode 100644 index 00000000000..8edf4e62b0f --- /dev/null +++ b/backend.native/konan.properties @@ -0,0 +1,21 @@ + +// TODO: utilize substitution mechanism from interop. + +sysRoot.osx = target-sysroot-1-darwin-macos +llvmHome.osx = clang+llvm-3.8.0-darwin-macos + +sysRoot.linux = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu/sysroot +llvmHome.linux = clang+llvm-3.8.0-linux-x86-64 +libGcc.linux = target-gcc-toolchain-3-linux-x86-64/lib/gcc/x86_64-unknown-linux-gnu/4.8.5 + +llvmLtoFlags.osx = -O3 -function-sections -exported-symbol=_main +llvmLlcFlags.osx = -mtriple=x86_64-apple-macosx10.10.0 +linkerOptimizationFlags.osx = -dead_strip +macosVersionMin.osx = 10.10.0 + +llvmLtoFlags.linux = -O3 -function-sections -exported-symbol=main +llvmLlcFlags.linux = -march=x86-64 +linkerKonanFlags.linux = -Bstatic -lc++abi -Bdynamic -ldl -lm -lpthread +linkerOptimizationFlags.linux = --gc-sections +pluginOptimizationFlags.linux = -plugin-opt=mcpu=x86-64 -plugin-opt=O3 + diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index f3ae8dc573c..18dccc25083 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -12,11 +12,12 @@ abstract class KonanTest extends DefaultTask { protected String source def backendNative = project.project(":backend.native") def runtimeProject = project.project(":runtime") + def dist = project.parent.parent.file("dist") def llvmLlc = llvmTool("llc") - def runtimeBc = new File("${runtimeProject.buildDir.canonicalPath}/runtime.bc") - def launcherBc = new File("${runtimeProject.buildDir.canonicalPath}/launcher.bc") - def startKtBc = new File("${runtimeProject.buildDir.canonicalPath}/start.kt.bc") - def stdlibKtBc = new File("${runtimeProject.buildDir.canonicalPath}/stdlib.kt.bc") + def runtimeBc = new File("${dist.canonicalPath}/lib/runtime.bc").absolutePath + def launcherBc = new File("${dist.canonicalPath}/lib/launcher.bc").absolutePath + def startKtBc = new File("${dist.canonicalPath}/lib/start.kt.bc").absolutePath + def stdlibKtBc = new File("${dist.canonicalPath}/lib/stdlib.kt.bc").absolutePath def mainC = 'main.c' String goldValue = null String testData = null @@ -29,50 +30,38 @@ abstract class KonanTest extends DefaultTask { } public KonanTest(){ - dependsOn([project.project(":runtime").tasks['build'], - project.parent.tasks['build'], - project.project(":backend.native").tasks['stdlib'], - project.project(":backend.native").tasks['start'] - ]) + // TODO: that's a long reach up the project tree. + // May be we should reorganize a little. + dependsOn(project.parent.parent.tasks['dist']) } + abstract void compileTest(String source, String exe) - - abstract void compileTest(File sourceS, File runtimeS, File libraryPath, File out) - - private File kt2bc(String ktSource) { - def sourceKt = project.file(ktSource) - def sourceBc = new File("${sourceKt.absolutePath}.bc") - println "${runtimeBc}" - + protected void runCompiler(String source, String output, List moreArgs) { project.javaexec { main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' classpath = project.configurations.cli_bc - jvmArgs "-ea" - args("-output", "${sourceBc.absolutePath}", - "-runtime", "${runtimeBc.absolutePath}", - "-library", "${stdlibKtBc.absolutePath}", - "${sourceKt.absolutePath}", + jvmArgs "-ea", + "-Dkonan.home=${dist.canonicalPath}", + "-Djava.library.path=${dist.canonicalPath}/konan/nativelib" + args("-output", output, + source, + *moreArgs, *project.globalArgs) - - String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs" - environment 'LD_LIBRARY_PATH' : libraryPath - environment 'DYLD_LIBRARY_PATH' : libraryPath } - return sourceBc } @TaskAction void executeTest() { - def sourceS = bc2s(kt2bc(source)) - def exe = new File(sourceS.absolutePath.replace(".kt.S", ".kt.exe")) - def libraryPath = new File("${runtimeProject.file('src/main/bc').absolutePath}") - compileTest(sourceS, bc2s(runtimeBc), stdlibKtBc, exe) - println "execution :${exe.absolutePath}" + def sourceKt = project.file(source).absolutePath + def exe = sourceKt.replace(".kt", ".kt.exe") + + compileTest(sourceKt, exe) + println "execution :$exe" def out = null project.exec { - commandLine "${exe.absolutePath}" + commandLine exe if (arguments != null) { args arguments } @@ -88,17 +77,6 @@ abstract class KonanTest extends DefaultTask { throw new RuntimeException("test failed") } - private File bc2s(File bcFile) { - def outputFile = new File("${bcFile.absolutePath.replace(".bc",".S")}") - println "${bcFile.absolutePath} -> ${outputFile.absolutePath}" - println "tool: ${llvmLlc}" - project.exec { - commandLine "${llvmLlc}", "-o", "${outputFile.absolutePath}" , "${bcFile}", - '-disable-fp-elim' // currently required for stack traces - } - return outputFile - } - private String llvmTool(String tool) { return "${project.llvmDir}/bin/${tool}" } @@ -109,89 +87,24 @@ abstract class KonanTest extends DefaultTask { } class RunKonanTest extends KonanTest { - void compileTest(File sourceS, File runtimeS, File libraryPath, File exe) { - project.execClang { - executable "clang" - def argList = [] - argList.addAll([ launcherBc.absolutePath, startKtBc.absolutePath, - runtimeS.absolutePath, stdlibKtBc.absolutePath, sourceS.absolutePath, "-o", exe.absolutePath ]) - argList.addAll(clangLinkArgs()) - args argList - } + void compileTest(String source, String exe) { + runCompiler(source, exe, []) } } class LinkKonanTest extends KonanTest { protected String lib - private File dir2bc(String ktSources) { - def sourcesKt = project.file(ktSources) - def libBc = new File("${sourcesKt.absolutePath}/libout.kt.bc") - def libPath = "${libBc.absolutePath}" - - project.javaexec { - main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath = project.configurations.cli_bc - jvmArgs "-ea" - args("-output", "${libPath}", - "-runtime", "${runtimeBc.absolutePath}", - sourcesKt, *project.globalArgs) - - String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs" - environment 'LD_LIBRARY_PATH' : libraryPath - environment 'DYLD_LIBRARY_PATH' : libraryPath - - } - return libBc + void compileTest(String source, String exe) { + def libDir = project.file(lib).absolutePath + def libBc = "${libDir}.bc" + + runCompiler(lib, libBc, ['-nolink', '-nostdlib']) + runCompiler(source, exe, ['-library', libBc]) } - - private File link(String ktSource, File libBc) { - def sourceKt = project.file(ktSource) - def outPath = "${sourceKt.absolutePath.replace('.kt', '.kt.bc')}" - def outBc = new File(outPath) - - project.javaexec { - main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath = project.configurations.cli_bc - jvmArgs "-ea" - args("-output", outPath, - "-runtime", "${runtimeBc.absolutePath}", - "-library", "${stdlibKtBc.absolutePath}", - "-library", "${libBc.absolutePath}", - "${sourceKt.absolutePath}", - *project.globalArgs) - - String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs" - environment 'LD_LIBRARY_PATH' : libraryPath - environment 'DYLD_LIBRARY_PATH' : libraryPath - - } - return outBc - } - - void compileTest(File sourceS, File runtimeS, File stdlibKtBc, File exe) { - def testC = sourceS.absolutePath.replace(".kt.S", "-test.c") - project.execClang { - executable "clang" - def argList = [] - argList.addAll([ runtimeS.absolutePath, stdlibKtBc.absolutePath, - sourceS.absolutePath, "-o", exe.absolutePath, testC, mainC ]) - argList.addAll(clangLinkArgs()) - args argList - } - } - - - @TaskAction - void executeTest() { - def libBc = dir2bc(lib) - def outBc = link(source, libBc) - } - - - } + task run() { dependsOn(tasks.withType(KonanTest).matching { it.enabled }) } @@ -418,6 +331,7 @@ task empty_string(type: RunKonanTest) { task intrinsic(type: RunKonanTest) { source = "codegen/function/intrinsic.kt" } + /* Disabled until we extract the classes that should be always present from stdlib.kt.bc into a separate binary. @@ -838,4 +752,4 @@ task expression_as_statement(type: RunKonanTest) { disabled = true goldValue = "Ok\n" source = "codegen/basics/expression_as_statement.kt" -} \ No newline at end of file +} diff --git a/build.gradle b/build.gradle index ec02dc6695f..71dbcea2d6b 100644 --- a/build.gradle +++ b/build.gradle @@ -179,36 +179,18 @@ task dist_compiler(type: Copy) { into('bin') } -} -task stdlib(type: Exec) { - dependsOn 'dist_compiler', 'list_dist' - - commandLine('./dist/bin/konanc', - "-J${jvmArgs.join(' ')}", - '-nolink', '-nostdlib', 'runtime/src/main/kotlin', '-o', - 'dist/lib/stdlib.kt.bc', *globalArgs) - - outputs.file(project.file('dist/lib/stdlib.kt.bc')) + from(project(':backend.native').file('konan.properties')) { + into('konan') + } } task list_dist(type: Exec) { commandLine 'find', 'dist' } -task start(type: Exec) { - dependsOn 'dist_compiler', 'stdlib' - - commandLine('./dist/bin/konanc', - "-J${jvmArgs.join(' ')}", - '-nolink', 'runtime/src/launcher/kotlin', '-o', - 'dist/lib/start.kt.bc', *globalArgs) - - outputs.file(project.file('dist/lib/start.kt.bc')) -} - task dist_runtime(type: Copy) { - dependsOn ':runtime:build' + dependsOn ':runtime:build', ':backend.native:stdlib', ':backend.native:start' destinationDir file('dist') @@ -219,7 +201,7 @@ task dist_runtime(type: Copy) { } task dist { - dependsOn 'dist_compiler', 'dist_runtime', 'stdlib', 'start' + dependsOn 'dist_compiler', 'dist_runtime' } task demo(type: Exec) { @@ -229,6 +211,8 @@ task demo(type: Exec) { } task clean { - delete 'dist' + doLast { + delete 'dist' + } } diff --git a/cmd/konanc b/cmd/konanc index 473586d780b..7dc12106dc7 100755 --- a/cmd/konanc +++ b/cmd/konanc @@ -1,15 +1,15 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash if [ -z "$JAVACMD" -a -n "$JAVA_HOME" -a -x "$JAVA_HOME/bin/java" ]; then JAVACMD="$JAVA_HOME/bin/java" else JAVACMD=java fi +[ -n "$JAVACMD" ] || JAVACMD=java +[ -n "$JAVA_OPTS" ] || JAVA_OPTS="-Xmx256M -Xms32M" declare -a java_args declare -a konan_args -declare -a clang_args -declare -a clang_opt_args while [ $# -gt 0 ]; do case "$1" in @@ -21,31 +21,8 @@ while [ $# -gt 0 ]; do java_args=("${java_args[@]}" "${1:2}") shift ;; - -library) - clang_args=("${clang_args[@]}" "${libraries[@]}" "$2") - konan_args=("${konan_args[@]}" "$1" "$2") - shift - shift - ;; - -nolink) - NOLINK=YES - shift - ;; - -nostdlib) - NOSTDLIB=YES - shift - ;; - -opt) - OPTIMIZE=YES - shift - ;; - -o|-output) - OUTPUT_NAME=$2 - shift - shift - ;; -X*) - clang_args=("${clang_args[@]}" "${1:2}") + echo "TODO: need to pass arguments to all the tools somehow." shift ;; *) @@ -55,10 +32,6 @@ while [ $# -gt 0 ]; do esac done -[ -n "$KONAN_COMPILER" ] || KONAN_COMPILER=org.jetbrains.kotlin.cli.bc.K2NativeKt -[ -n "$JAVACMD" ] || JAVACMD=java -[ -n "$JAVA_OPTS" ] || JAVA_OPTS="-Xmx256M -Xms32M" - # Based on findScalaHome() from scalac script findHome() { local source="${BASH_SOURCE[0]}" @@ -72,76 +45,19 @@ findHome() { KONAN_HOME="$(findHome)" -echo $KONAN_HOME - KONAN_JAR="${KONAN_HOME}/konan/lib/backend.native.jar" KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar" INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar" -PROTOBUF_JAR="${KONAN_HOME}/konan/lib/protobuf-java-3.0.0.jar" - -KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$PROTOBUF_JAR:$KONAN_JAR" - NATIVE_LIB="${KONAN_HOME}/konan/nativelib" -DYLD=$DYLD_LIBRARY_PATH:$NATIVE_LIB +KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR" +KONAN_COMPILER=org.jetbrains.kotlin.cli.bc.K2NativeKt -RUNTIME="$KONAN_HOME/lib/runtime.bc" -STDLIB="$KONAN_HOME/lib/stdlib.kt.bc" # # KONAN BACKEND INVOCATION # -java_args=("${java_args[@]} -noverify -Djava.library.path=${KONAN_HOME}/konan/nativelib") +java_args=("${java_args[@]} -noverify -Dkonan.home=${KONAN_HOME} -Djava.library.path=${NATIVE_LIB}") -konan_args=("-runtime" $RUNTIME "${konan_args[@]}") -[ -z $NOSTDLIB ] && konan_args=("-library" $STDLIB "${konan_args[@]}") +$JAVACMD $JAVA_OPTS ${java_args[@]} -cp $KONAN_CLASSPATH $KONAN_COMPILER ${konan_args[@]} -if [ -z $OUTPUT_NAME ] ; then - if [ -z $NOLINK ] ; then - OUTPUT_NAME="program.kexe" - KTBC_NAME="program.kt.bc" - else - KTBC_NAME="program.kt.bc" - fi -else - if [ -z $NOLINK ] ; then - KTBC_NAME="${OUTPUT_NAME}.kt.bc" - else - KTBC_NAME="${OUTPUT_NAME}" - fi -fi - -DYLD_LIBRARY_PATH=$DYLD LD_LIBRARY_PATH=$DYLD $JAVACMD $JAVA_OPTS ${java_args[@]} -cp $KONAN_CLASSPATH $KONAN_COMPILER -output ${KTBC_NAME} ${konan_args[@]} - -COMPILER_EXIT_CODE="$?" -[ -z $NOLINK ] || exit $COMPILER_EXIT_CODE - -# -# LINK STAGE -# - -# TODO: We should probably copy the dependencies into dist. -DEPENDENCIES="$KONAN_HOME/../dependencies/all" - -LAUNCHER="${KONAN_HOME}/lib/launcher.bc" -START="${KONAN_HOME}/lib/start.kt.bc" - -# We filter this script during installation -# substituting the proper platform dependent arguments here. -CLANG_PLATFORM_ARGS="FILTER_CLANG_PLATFORM_ARGS" -CLANG_PLATFORM_OPT="FILTER_CLANG_PLATFORM_OPT" -CLANG_BIN_PATH="FILTER_CLANG_BIN_PATH" - -clang_opt_args=("$CLANG_PLATFORM_OPT") - -CLANG="$CLANG_BIN_PATH/clang" - -# TODO: Compilation without stdlib.kt.bc doesn't work because 'start' requires stdlib, for example. -# Probably, we need a .klib compilation mode. -# [ -z $NOSTDLIB ] && clang_args=($STDLIB "${clang_args[@]}") -[ "$OPTIMIZE" == "YES" ] && clang_args=("${clang_opt_args[@]} ${clang_args[@]}") -clang_args=($STDLIB "${clang_args[@]}") -clang_args=($LAUNCHER $START $RUNTIME "${clang_args[@]}") -clang_args=("${clang_args[@]} $CLANG_PLATFORM_ARGS") - -$CLANG $KTBC_NAME -o $OUTPUT_NAME ${clang_args[@]}