Move everything under kotlin-native folder

I was forced to manually do update the following files, because otherwise
they would be ignored according .gitignore settings. Probably they
should be deleted from repo.

Interop/.idea/compiler.xml
Interop/.idea/gradle.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_runtime_1_0_3.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_stdlib_1_0_3.xml
Interop/.idea/modules.xml
Interop/.idea/modules/Indexer/Indexer.iml
Interop/.idea/modules/Runtime/Runtime.iml
Interop/.idea/modules/StubGenerator/StubGenerator.iml
backend.native/backend.native.iml
backend.native/bc.frontend/bc.frontend.iml
backend.native/cli.bc/cli.bc.iml
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt
backend.native/tests/link/lib/foo.kt
backend.native/tests/link/lib/foo2.kt
backend.native/tests/teamcity-test.property
This commit is contained in:
Stanislav Erokhin
2020-10-27 21:00:28 +03:00
parent 91e4162dad
commit f624800b84
2830 changed files with 0 additions and 0 deletions
@@ -0,0 +1,30 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
buildscript {
apply from: "$rootDir/gradle/kotlinGradlePlugin.gradle"
}
apply plugin: 'kotlin'
repositories {
maven {
url buildKotlinCompilerRepo
}
}
compileKotlin {
kotlinOptions {
freeCompilerArgs = ['-Xskip-metadata-version-check']
}
}
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
implementation project(':backend.native')
implementation project(':Interop:StubGenerator')
implementation project(':klib')
implementation project(":utilities:basic-utils")
}
@@ -0,0 +1,421 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.cli.bc.K2Native
import org.jetbrains.kotlin.konan.file.File
import java.util.concurrent.*
import kotlinx.cli.*
import org.jetbrains.kotlin.backend.konan.CachedLibraries
import org.jetbrains.kotlin.backend.konan.OutputFiles
import org.jetbrains.kotlin.backend.konan.files.renameAtomic
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.customerDistribution
import org.jetbrains.kotlin.konan.util.KonanHomeProvider
import org.jetbrains.kotlin.konan.util.PlatformLibsInfo
import org.jetbrains.kotlin.konan.util.visibleName
import org.jetbrains.kotlin.native.interop.tool.CommonInteropArguments.Companion.DEFAULT_MODE
import org.jetbrains.kotlin.native.interop.tool.CommonInteropArguments.Companion.MODE_METADATA
import org.jetbrains.kotlin.native.interop.tool.CommonInteropArguments.Companion.MODE_SOURCECODE
import org.jetbrains.kotlin.native.interop.tool.SHORT_MODULE_NAME
import java.io.PrintWriter
import java.io.StringWriter
import java.util.concurrent.atomic.AtomicInteger
import kotlin.system.exitProcess
import java.io.File as JFile
// TODO: We definitely need to unify logging in different parts of the compiler.
private class Logger(val level: Level = Level.NORMAL) {
fun log(message: String) {
println(message)
}
fun verbose(message: String) {
if (level == Level.VERBOSE) {
println(message)
}
}
enum class Level {
NORMAL, VERBOSE
}
}
private fun Logger.logFailedLibraries(built: Map<DefFile, ProcessingStatus>) {
log("Processing platform libraries finished with errors.")
built.forEach { (def, status) ->
if (status is ProcessingStatus.FAIL) {
log(" ${def.name}: ${status.error}")
}
}
}
private fun Logger.logStackTrace(error: Throwable) {
val stringWriter = StringWriter()
error.printStackTrace(PrintWriter(stringWriter))
verbose(stringWriter.toString())
}
// TODO: Use Distribution's paths after compiler update.
fun generatePlatformLibraries(args: Array<String>) {
// IMPORTANT! These command line keys are used by the Gradle plugin to configure platform libraries generation,
// so any changes in them must be reflected at the Gradle plugin side too.
// See org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator in the Big Kotlin repo.
val argParser = ArgParser("generate-platform", prefixStyle = ArgParser.OptionPrefixStyle.JVM)
val inputDirectoryPath by argParser.option(
ArgType.String,
"input-directory", "i",
"Input directory. Default value is <dist>/konan/platformDef/<target>"
)
val outputDirectoryPath by argParser.option(
ArgType.String,
"output-directory", "o",
"Output directory. Default value is <dist>/klib/platform/<target>"
)
val targetName by argParser.option(
ArgType.String, "target", "t", "Compilation target").required()
val saveTemps by argParser.option(
ArgType.Boolean, "save-temps", "s", "Save temporary files").default(false)
val stdlibPath by argParser.option(
ArgType.String,
"stdlib-path", "S",
"Place where stdlib is located. Default value is <dist>/klib/common/stdlib"
)
val dynamicCacheKind = CompilerOutputKind.DYNAMIC_CACHE.visibleName
val staticCacheKind = CompilerOutputKind.STATIC_CACHE.visibleName
val cacheKind by argParser.option(
ArgType.Choice(listOf(dynamicCacheKind, staticCacheKind)), "cache-kind", "k", "Type of cache."
).default(dynamicCacheKind)
val cacheDirectoryPath by argParser.option(
ArgType.String, "cache-directory", "c", "Cache output directory")
val mode by argParser.option(
ArgType.Choice(listOf(MODE_METADATA, MODE_SOURCECODE)),
fullName = "mode",
shortName = "m",
description = "The way interop library is generated."
).default(DEFAULT_MODE)
val verbose by argParser.option(
ArgType.Boolean,
"verbose", "v",
"Show verbose log messages"
).default(false)
val cacheArgs by argParser.option(
ArgType.String, "cache-arg",
description = "An argument passed to compiler during cache building. Used only if -cache-directory is specified."
).multiple()
val rebuild by argParser.option(
ArgType.Boolean, fullName = "rebuild", description = "Rebuild already existing libraries"
).default(false)
argParser.parse(args)
val distribution = customerDistribution(KonanHomeProvider.determineKonanHome())
val target = HostManager(distribution).targetByName(targetName)
val inputDirectory = inputDirectoryPath?.File()
?: File(distribution.konanSubdir, "platformDef").child(target.visibleName)
val outputDirectory = outputDirectoryPath?.File()
?: File(distribution.klib, "platform").child(target.visibleName)
val cacheDirectory = cacheDirectoryPath?.File()
if (!inputDirectory.exists) throw Error("input directory doesn't exist")
if (!outputDirectory.exists) {
outputDirectory.mkdirs()
}
if (cacheDirectory != null && !cacheDirectory.exists) {
cacheDirectory.mkdirs()
}
val stdlibFile = stdlibPath?.File() ?: File(distribution.stdlib)
val logger = Logger(if (verbose) Logger.Level.VERBOSE else Logger.Level.NORMAL)
val cacheInfo = cacheDirectory?.let { CacheInfo(it, cacheKind, cacheArgs) }
generatePlatformLibraries(
target, mode,
DirectoriesInfo(inputDirectory, outputDirectory, stdlibFile), cacheInfo,
rebuild, saveTemps, logger
)
}
private sealed class ProcessingStatus {
object WAIT: ProcessingStatus()
object SUCCESS: ProcessingStatus()
object FAILED_DEPENDENCIES: ProcessingStatus()
class FAIL(val error: Throwable) : ProcessingStatus()
}
private data class DirectoriesInfo(val inputDirectory: File, val outputDirectory: File, val stdlib: File)
private data class CacheInfo(val cacheDirectory: File, val cacheKind: String, val cacheArgs: List<String>)
private class DefFile(val name: String, val depends: MutableList<DefFile>) {
override fun toString(): String = "$name: [${depends.joinToString(separator = ", ") { it.name }}]"
val libraryName: String
get() = "${PlatformLibsInfo.namePrefix}$name"
val shortLibraryName: String
get() = name
}
private fun createTempDir(prefix: String, parent: File): File =
File(createTempDir(prefix, directory = JFile(parent.absolutePath)).absolutePath)
private fun File.deleteAtomicallyIfPossible(tmpDirectory: File) {
// Try to atomically delete the old directory.
val tmpToDelete = createTempFile(directory = JFile(tmpDirectory.absolutePath))
if (renameAtomic(this.absolutePath, tmpToDelete.absolutePath, replaceExisting = true)) {
tmpToDelete.deleteRecursively()
} else {
// Can't move to a tmp directory -> delete in a regular way.
this.deleteRecursively()
}
}
private fun topoSort(defFiles: List<DefFile>): List<DefFile> {
// Do DFS toposort.
val markGray = mutableSetOf<DefFile>()
val markBlack = mutableSetOf<DefFile>()
val result = mutableListOf<DefFile>()
fun visit(def: DefFile) {
if (markBlack.contains(def)) return
if (markGray.contains(def)) throw Error("$def is part of cycle")
markGray += def
def.depends.forEach {
visit(it)
}
markGray -= def
markBlack += def
result += def
}
var index = 0
while (markBlack.size < defFiles.size) {
visit(defFiles[index++])
}
return result
}
private fun generateLibrary(
target: KonanTarget,
mode: String,
def: DefFile,
directories: DirectoriesInfo,
tmpDirectory: File,
rebuild: Boolean,
logger: Logger
) = with(directories) {
val defFile = inputDirectory.child("${def.name}.def")
val outKlib = outputDirectory.child(def.libraryName)
if (outKlib.exists && !rebuild) {
logger.verbose("Skip generating ${def.name} as it's already generated")
return
}
val tmpKlib = tmpDirectory.child(def.libraryName)
try {
val cinteropArgs = arrayOf(
"-o", tmpKlib.absolutePath,
"-target", target.visibleName,
"-def", defFile.absolutePath,
"-compiler-option", "-fmodules-cache-path=${tmpDirectory.child("clangModulesCache").absolutePath}",
"-repo", outputDirectory.absolutePath,
"-no-default-libs", "-no-endorsed-libs", "-Xpurge-user-libs", "-nopack",
"-mode", mode,
"-$SHORT_MODULE_NAME", def.shortLibraryName,
*def.depends.flatMap { listOf("-l", "$outputDirectory/${it.libraryName}") }.toTypedArray()
)
logger.verbose("Run cinterop with args: ${cinteropArgs.joinToString(separator = " ")}")
invokeInterop("native", cinteropArgs)?.let { K2Native.mainNoExit(it) }
if (rebuild) {
outKlib.deleteAtomicallyIfPossible(tmpDirectory)
}
// Atomically move the generated library to the destination path.
if (!renameAtomic(tmpKlib.absolutePath, outKlib.absolutePath, replaceExisting = false)) {
tmpKlib.deleteRecursively()
}
} finally {
tmpKlib.deleteRecursively()
}
}
private fun getLibraryCacheDir(
libraryName: String,
target: KonanTarget,
cacheDirectory: File,
cacheKind: String
): File {
val cacheBaseName = CachedLibraries.getCachedLibraryName(libraryName)
val cacheOutputKind = CompilerOutputKind.valueOf(cacheKind.toUpperCase())
return OutputFiles(cacheDirectory.child(cacheBaseName).absolutePath, target, cacheOutputKind).mainFile.File()
}
private fun buildCache(
target: KonanTarget,
def: DefFile,
outputDirectory: File,
cacheInfo: CacheInfo,
rebuild: Boolean,
logger: Logger
) = with(cacheInfo) {
val libraryCacheDir = getLibraryCacheDir(def.name, target, cacheDirectory, cacheKind)
if (libraryCacheDir.listFilesOrEmpty.isNotEmpty() && !rebuild) {
logger.verbose("Skip precompiling ${def.name} as it's already precompiled")
return
}
if (rebuild) {
libraryCacheDir.deleteRecursively()
}
val compilerArgs = arrayOf(
"-p", cacheKind,
"-target", target.visibleName,
"-repo", outputDirectory.absolutePath,
"-Xadd-cache=${outputDirectory.absolutePath}/${def.libraryName}",
"-Xcache-directory=${cacheDirectory.absolutePath}",
*cacheArgs.toTypedArray()
)
logger.verbose("Run compiler with args: ${compilerArgs.joinToString(separator = " ")}")
K2Native.mainNoExit(compilerArgs)
}
private fun buildStdlibCache(
target: KonanTarget,
stdlib: File,
cacheInfo: CacheInfo,
logger: Logger
) = with(cacheInfo) {
val stdlibCacheFile = getLibraryCacheDir("stdlib", target, cacheDirectory, cacheKind)
if (stdlibCacheFile.exists) {
logger.verbose("Skip precompiling standard library as it's already precompiled")
return
}
logger.log("Precompiling standard library...")
val compilerArgs = arrayOf(
"-p", cacheKind,
"-target", target.visibleName,
"-Xadd-cache=${stdlib.absolutePath}",
"-Xcache-directory=${cacheDirectory.absolutePath}",
*cacheArgs.toTypedArray()
)
logger.verbose("Run compiler with args: ${compilerArgs.joinToString(separator = " ")}")
K2Native.mainNoExit(compilerArgs)
}
private fun generatePlatformLibraries(target: KonanTarget, mode: String,
directories: DirectoriesInfo, cacheInfo: CacheInfo?,
rebuild: Boolean, saveTemps: Boolean, logger: Logger) = with(directories) {
if (cacheInfo != null) {
buildStdlibCache(target, stdlib, cacheInfo, logger)
}
logger.verbose("Generating platform libraries from $inputDirectory to $outputDirectory for ${target.visibleName}")
if (cacheInfo != null) {
logger.verbose("Precompiling platform libraries to ${cacheInfo.cacheDirectory} (cache kind: ${cacheInfo.cacheKind})")
}
val tmpDirectory = createTempDir("build-", outputDirectory)
// Delete the tmp directory in case of execution interruption.
val deleteTmpHook = Thread {
if (!saveTemps) {
tmpDirectory.deleteRecursively()
}
}
Runtime.getRuntime().addShutdownHook(deleteTmpHook)
// Build dependencies graph.
val defFiles = mutableMapOf<String, DefFile>()
val dependsRegex = Regex("^depends = (.*)")
inputDirectory.listFilesOrEmpty.filter { it.extension == "def" }.forEach { file ->
val name = file.name.split(".").also { assert(it.size == 2) }[0]
val def = defFiles.getOrPut(name) {
DefFile(name, mutableListOf())
}
file.forEachLine { line ->
val match = dependsRegex.matchEntire(line)
if (match != null) {
match.groupValues[1].split(" ").forEach { dependency ->
def.depends.add(defFiles.getOrPut(dependency) {
DefFile(dependency, mutableListOf())
})
}
}
}
}
val sorted = topoSort(defFiles.values.toList())
val numCores = Runtime.getRuntime().availableProcessors()
val executorPool = ThreadPoolExecutor(numCores, numCores,
10, TimeUnit.SECONDS, ArrayBlockingQueue(1000),
Executors.defaultThreadFactory(), RejectedExecutionHandler { r, _ ->
logger.log("Execution rejected: $r")
throw Error("Must not happen!")
})
val built = ConcurrentHashMap(sorted.associateWith<DefFile, ProcessingStatus> { ProcessingStatus.WAIT })
// Now run interop tool on toposorted dependencies.
val countTotal = sorted.size
val countProcessed = AtomicInteger(0)
try {
tmpDirectory.mkdirs()
sorted.forEach { def ->
executorPool.execute {
// A bit ugly, we just block here until all dependencies are built.
while (def.depends.any { built[it] == ProcessingStatus.WAIT }) {
Thread.sleep(100)
}
try {
if (def.depends.any { built[it] is ProcessingStatus.FAIL }) {
built[def] = ProcessingStatus.FAILED_DEPENDENCIES
return@execute
}
logger.log("Processing ${def.name} (${countProcessed.incrementAndGet()}/$countTotal)...")
generateLibrary(target, mode, def, directories, tmpDirectory, rebuild, logger)
if (cacheInfo != null) {
buildCache(target, def, outputDirectory, cacheInfo, rebuild, logger)
}
built[def] = ProcessingStatus.SUCCESS
} catch (e: Throwable) {
built[def] = ProcessingStatus.FAIL(e)
logger.logStackTrace(e)
}
}
}
executorPool.shutdown()
executorPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS)
if (built.values.any { it != ProcessingStatus.SUCCESS }) {
logger.logFailedLibraries(built)
exitProcess(-1)
}
} finally {
if (!saveTemps) {
tmpDirectory.deleteRecursively()
}
Runtime.getRuntime().removeShutdownHook(deleteTmpHook)
}
}
@@ -0,0 +1,78 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.cli.bc.SHORT_MODULE_NAME_ARG
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.konan.util.KonanHomeProvider
import org.jetbrains.kotlin.native.interop.gen.jvm.InternalInteropOptions
import org.jetbrains.kotlin.native.interop.gen.jvm.interop
import org.jetbrains.kotlin.native.interop.tool.*
// TODO: this function should eventually be eliminated from 'utilities'.
// The interaction of interop and the compiler should be streamlined.
/**
* @return null if there is no need in compiler invocation.
* Otherwise returns array of compiler args.
*/
fun invokeInterop(flavor: String, args: Array<String>): Array<String>? {
val arguments = if (flavor == "native") CInteropArguments() else JSInteropArguments()
arguments.argParser.parse(args)
val outputFileName = arguments.output
val noDefaultLibs = arguments.nodefaultlibs || arguments.nodefaultlibsDeprecated
val noEndorsedLibs = arguments.noendorsedlibs
val purgeUserLibs = arguments.purgeUserLibs
val nopack = arguments.nopack
val temporaryFilesDir = arguments.tempDir
val moduleName = (arguments as? CInteropArguments)?.moduleName
val shortModuleName = (arguments as? CInteropArguments)?.shortModuleName
val buildDir = File("$outputFileName-build")
val generatedDir = File(buildDir, "kotlin")
val nativesDir = File(buildDir,"natives")
val manifest = File(buildDir, "manifest.properties")
val cstubsName ="cstubs"
val libraries = arguments.library
val repos = arguments.repo
val targetRequest = if (arguments is CInteropArguments) arguments.target
else (arguments as JSInteropArguments).target
val target = PlatformManager(KonanHomeProvider.determineKonanHome()).targetManager(targetRequest).target
val cinteropArgsToCompiler = interop(flavor, args,
InternalInteropOptions(generatedDir.absolutePath,
nativesDir.absolutePath,manifest.path,
cstubsName.takeIf { flavor == "native" }
)
) ?: return null // There is no need in compiler invocation if we're generating only metadata.
val nativeStubs =
if (flavor == "wasm")
arrayOf("-include-binary", File(nativesDir, "js_stubs.js").path)
else
arrayOf("-native-library", File(nativesDir, "$cstubsName.bc").path)
return arrayOf(
generatedDir.path,
"-produce", "library",
"-o", outputFileName,
"-target", target.visibleName,
"-manifest", manifest.path,
"-Xtemporary-files-dir=$temporaryFilesDir") +
nativeStubs +
cinteropArgsToCompiler +
libraries.flatMap { listOf("-library", it) } +
repos.flatMap { listOf("-repo", it) } +
(if (noDefaultLibs) arrayOf("-$NODEFAULTLIBS") else emptyArray()) +
(if (noEndorsedLibs) arrayOf("-$NOENDORSEDLIBS") else emptyArray()) +
(if (purgeUserLibs) arrayOf("-$PURGE_USER_LIBS") else emptyArray()) +
(if (nopack) arrayOf("-$NOPACK") else emptyArray()) +
moduleName?.let { arrayOf("-module-name", it) }.orEmpty() +
shortModuleName?.let { arrayOf("$SHORT_MODULE_NAME_ARG=$it") }.orEmpty() +
arguments.kotlincOption
}
@@ -0,0 +1,43 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.konan.exec.Command
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.konan.util.KonanHomeProvider
fun runLlvmTool(args: Array<String>) {
val toolName = args[0]
val toolArguments = args.drop(1)
val platform = platformManager().hostPlatform
val llvmHome = platform.configurables.absoluteLlvmHome
val toolPath = "$llvmHome/bin/$toolName"
runCommand(toolPath, *toolArguments.toTypedArray())
}
fun runLlvmClangToolWithTarget(args: Array<String>) {
val toolName = args[0]
val targetName = args[1]
val toolArguments = args.drop(2)
val platformManager = platformManager()
val platform = platformManager.platform(platformManager.targetByName(targetName))
val llvmHome = platform.configurables.absoluteLlvmHome
val toolPath = "$llvmHome/bin/$toolName"
runCommand(toolPath, *platform.clang.clangArgs, *toolArguments.toTypedArray())
}
private fun platformManager() = PlatformManager(KonanHomeProvider.determineKonanHome())
private fun runCommand(vararg args: String) {
Command(*args)
.logWith { println(it()) }
.execute()
}
@@ -0,0 +1,44 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.native.interop.gen.defFileDependencies
import org.jetbrains.kotlin.cli.bc.main as konancMain
import org.jetbrains.kotlin.cli.klib.main as klibMain
import org.jetbrains.kotlin.cli.bc.mainNoExitWithGradleRenderer as konancMainForGradle
private fun mainImpl(args: Array<String>, konancMain: (Array<String>) -> Unit) {
val utilityName = args[0]
val utilityArgs = args.drop(1).toTypedArray()
when (utilityName) {
"konanc" ->
konancMain(utilityArgs)
"cinterop" -> {
val konancArgs = invokeInterop("native", utilityArgs)
konancArgs?.let { konancMain(it) }
}
"jsinterop" -> {
val konancArgs = invokeInterop("wasm", utilityArgs)
konancArgs?.let { konancMain(it) }
}
"klib" ->
klibMain(utilityArgs)
"defFileDependencies" ->
defFileDependencies(utilityArgs)
"generatePlatformLibraries" ->
generatePlatformLibraries(utilityArgs)
"llvm" -> runLlvmTool(utilityArgs)
"clang" -> runLlvmClangToolWithTarget(utilityArgs)
else ->
error("Unexpected utility name")
}
}
fun main(args: Array<String>) = mainImpl(args, ::konancMain)
fun daemonMain(args: Array<String>) = mainImpl(args, ::konancMainForGradle)