Do not save temporary files by default

This commit is contained in:
Sergey Bogolepov
2018-02-21 11:29:49 +03:00
committed by Sergey Bogolepov
parent 049d29af75
commit 1be56f1e51
16 changed files with 172 additions and 121 deletions
+2 -5
View File
@@ -14,8 +14,6 @@
* limitations under the License.
*/
import org.jetbrains.kotlin.konan.target.HostManager
buildscript {
apply from: "$rootDir/gradle/kotlinGradlePlugin.gradle"
}
@@ -70,7 +68,7 @@ model {
clangstubs(NativeLibrarySpec) {
sources {
c.source.srcDir 'prebuilt/nativeInteropStubs/c'
cpp.source.srcDir 'prebuilt/nativeInteropStubs/cpp'
cpp.source.srcDir 'prebuilt/nativeInteropStubs/cpp'
}
binaries.all {
@@ -130,7 +128,6 @@ kotlinNativeInterop {
defFile 'clang.def'
compilerOpts cflags
linkerOpts ldflags
genTask.args '-keepcstubs', 'true'
genTask.dependsOn libclangextTask
genTask.inputs.dir libclangextDir
}
@@ -153,7 +150,7 @@ task updatePrebuilt {
}
copy {
from("$buildDir/nativelibs/${HostManager.host.visibleName}") {
from("$buildDir/interopTemp") {
include 'clangstubs.c'
}
into 'prebuilt/nativeInteropStubs/c'
@@ -18,6 +18,8 @@ package org.jetbrains.kotlin.native.interop.tool
import org.jetbrains.kotlin.cli.common.arguments.*
// TODO: unify camel and snake cases.
// Possible solution is to accept both cases
open class CommonInteropArguments : CommonToolArguments() {
@Argument(value = "-flavor", valueDescription = "<flavor>", description = "One of: jvm, native or wasm")
var flavor: String? = null
@@ -28,17 +30,20 @@ open class CommonInteropArguments : CommonToolArguments() {
@Argument(value = "-generated", valueDescription = "<dir>", description = "place generated bindings to the directory")
var generated: String? = null
@Argument(value = "-libraryPath", valueDescription = "<dir>", description = "add a library search path")
var libraryPath: Array<String> = arrayOf()
@Argument(value = "-manifest", valueDescription = "<file>", description = "library manifest addend")
var manifest: String? = null
@Argument(value = "-natives", valueDescription = "<directory>", description = "where to put the built native files")
var natives: String? = null
@Argument(value = "-manifest", valueDescription = "<file>", description = "library manifest addend")
var manifest: String? = null
@Argument(value = "-staticLibrary", valueDescription = "<file>", description = "embed static library to the result")
var staticLibrary: Array<String> = arrayOf()
@Argument(value = "-libraryPath", valueDescription = "<dir>", description = "add a library search path")
var libraryPath: Array<String> = arrayOf()
@Argument(value = "-temporaryFilesDir", valueDescription = "<dir>", description = "Save temporary files to the given directory")
var temporaryFilesDir: String? = null
}
class CInteropArguments : CommonInteropArguments() {
@@ -73,9 +78,6 @@ class CInteropArguments : CommonInteropArguments() {
var linker: String? = null
@Argument(value = "-cstubsname", valueDescription = "<name>", description = "provide a name for the generated c stubs file")
var cstubsname: String? = null
@Argument(value = "-keepcstubs", description = "preserve the generated c stubs for inspection")
var keepcstubs: Boolean = false
}
const val HEADER_FILTER_ADDITIONAL_SEARCH_PREFIX = "-headerFilterAdditionalSearchPrefix"
@@ -16,6 +16,8 @@
package org.jetbrains.kotlin.native.interop.gen.jvm
import org.jetbrains.kotlin.konan.TempFiles
import org.jetbrains.kotlin.konan.exec.Command
import org.jetbrains.kotlin.konan.util.DefFile
import org.jetbrains.kotlin.native.interop.gen.HeadersInclusionPolicyImpl
import org.jetbrains.kotlin.native.interop.gen.ImportsImpl
@@ -64,13 +66,6 @@ private fun substitute(properties: Properties, substitutions: Map<String, String
}
}
private fun ProcessBuilder.runExpectingSuccess() {
val res = this.start().waitFor()
if (res != 0) {
throw Error("Process finished with non-zero exit code: $res")
}
}
private fun <T> Collection<T>.atMostOne(): T? {
return when (this.size) {
0 -> null
@@ -84,35 +79,9 @@ private fun List<String>?.isTrue(): Boolean {
return this?.last() == "true"
}
private fun runCmd(command: Array<String>, workDir: File, verbose: Boolean = false) {
val builder = ProcessBuilder(*command)
.directory(workDir)
val logFile: File?
if (verbose) {
println(command.joinToString(" "))
builder.inheritIO()
logFile = null
} else {
logFile = createTempFile(suffix = ".log")
logFile.deleteOnExit()
builder.redirectOutput(ProcessBuilder.Redirect.to(logFile))
.redirectErrorStream(true)
}
try {
builder.runExpectingSuccess()
} catch (e: Throwable) {
if (!verbose) {
println(command.joinToString(" "))
logFile!!.useLines {
it.forEach { println(it) }
}
}
throw e
private fun runCmd(command: Array<String>, verbose: Boolean = false) {
Command(*command).getOutputLines(true).let { lines ->
if (verbose) lines.forEach(::println)
}
}
@@ -241,7 +210,7 @@ private fun processCLib(args: Array<String>): Array<String>? {
val flavor = KotlinPlatform.values().single { it.name.equals(flavorName, ignoreCase = true) }
val defFile = arguments.def?.let { File(it) }
val manifestAddend = arguments.manifest?.let { File(it) }
if (defFile == null && arguments.pkg == null) {
usage()
return null
@@ -309,6 +278,8 @@ private fun processCLib(args: Array<String>): Array<String>? {
val libName = arguments.cstubsname ?: fqParts.joinToString("") + "stubs"
val tempFiles = TempFiles(libName, arguments.temporaryFilesDir)
val headerFilterGlobs = def.config.headerFilter
val imports = parseImports(arguments.import)
val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
@@ -340,10 +311,10 @@ private fun processCLib(args: Array<String>): Array<String>? {
outKtFile.parentFile.mkdirs()
File(nativeLibsDir).mkdirs()
val outCFile = File("$nativeLibsDir/$libName.${language.sourceFileExtension}") // TODO: select the better location.
val outCFile = tempFiles.create(libName, ".${language.sourceFileExtension}")
outKtFile.bufferedWriter().use { ktFile ->
outCFile.bufferedWriter().use { cFile ->
File(outCFile.absolutePath).bufferedWriter().use { cFile ->
gen.generateFiles(ktFile = ktFile, cFile = cFile, entryPoint = entryPoint)
}
}
@@ -353,7 +324,7 @@ private fun processCLib(args: Array<String>): Array<String>? {
def.manifestAddendProperties.putAndRunOnReplace("package", outKtPkg) {
_, oldValue, newValue ->
warn("The package value `$oldValue` specified in .def file is overriden with explicit $newValue")
warn("The package value `$oldValue` specified in .def file is overridden with explicit $newValue")
}
def.manifestAddendProperties["interop"] = "true"
@@ -363,16 +334,14 @@ private fun processCLib(args: Array<String>): Array<String>? {
manifestAddend?.parentFile?.mkdirs()
manifestAddend?.let { def.manifestAddendProperties.storeProperties(it) }
val workDir = defFile?.absoluteFile?.parentFile ?: File(userDir)
if (flavor == KotlinPlatform.JVM) {
val outOFile = createTempFile(suffix = ".o")
val outOFile = tempFiles.create(libName,".o")
val compilerCmd = arrayOf(compiler, *gen.libraryForCStubs.compilerArgs.toTypedArray(),
"-c", outCFile.absolutePath, "-o", outOFile.absolutePath)
runCmd(compilerCmd, workDir, verbose)
runCmd(compilerCmd, verbose)
val outLib = File(nativeLibsDir, System.mapLibraryName(libName))
@@ -380,21 +349,14 @@ private fun processCLib(args: Array<String>): Array<String>? {
outOFile.absolutePath, "-shared", "-o", outLib.absolutePath,
*linkerOpts)
runCmd(linkerCmd, workDir, verbose)
outOFile.delete()
runCmd(linkerCmd, verbose)
} else if (flavor == KotlinPlatform.NATIVE) {
val outBcName = libName + ".bc"
val outLib = File(nativeLibsDir, outBcName)
val compilerCmd = arrayOf(compiler, *gen.libraryForCStubs.compilerArgs.toTypedArray(),
"-emit-llvm", "-c", outCFile.absolutePath, "-o", outLib.absolutePath)
runCmd(compilerCmd, workDir, verbose)
runCmd(compilerCmd, verbose)
}
if (!arguments.keepcstubs) {
outCFile.delete()
}
return argsToCompiler(staticLibraries, libraryPaths)
}
@@ -124,6 +124,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
arguments.mainPackage ?.let{ put(ENTRY, it) }
arguments.manifestFile ?.let{ put(MANIFEST_FILE, it) }
arguments.runtimeFile ?.let{ put(RUNTIME_FILE, it) }
arguments.temporaryFilesDir?.let { put(TEMPORARY_FILES_DIR, it) }
put(LIST_TARGETS, arguments.listTargets)
put(OPTIMIZATION, arguments.optimization)
@@ -119,8 +119,11 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
var purgeUserLibs: Boolean = false
@Argument(value = "--runtime", valueDescription = "<path>", description = "Override standard 'runtime.bc' location")
var runtimeFile: String? = null
@Argument(value = "--temporary_files_dir", valueDescription = "<path>", description = "Save temporary files to the given directory")
var temporaryFilesDir: String? = null
@Argument(value = "--time", description = "Report execution time for compiler phases")
var timePhases: Boolean = false
@@ -710,7 +710,7 @@ internal class CAdapterGenerator(
val exportedSymbols = mutableListOf<String>()
private fun makeGlobalStruct(top: ExportedElementScope) {
val headerFile = context.config.tempFiles.cAdapterHeader
val headerFile = context.config.outputFiles.cAdapterHeader
outputStreamWriter = headerFile.printWriter()
val exportedSymbol = "${prefix}_symbols"
@@ -19,7 +19,6 @@ import llvm.LLVMLinkModules2
import llvm.LLVMWriteBitcodeToFile
import org.jetbrains.kotlin.backend.konan.library.impl.buildLibrary
import org.jetbrains.kotlin.backend.konan.llvm.parseBitcodeFile
import org.jetbrains.kotlin.backend.konan.util.getValueOrNull
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
val CompilerOutputKind.isNativeBinary: Boolean get() = when (this) {
@@ -68,7 +67,7 @@ internal fun produceOutput(context: Context) {
LLVMWriteBitcodeToFile(llvmModule, output)
}
CompilerOutputKind.LIBRARY -> {
val output = context.config.outputName
val output = context.config.outputFiles.outputName
val libraryName = context.config.moduleId
val neededLibraries
= context.llvm.librariesForLibraryManifest
@@ -29,9 +29,9 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.konan.TempFiles
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.*
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
@@ -64,17 +64,14 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
val indirectBranchesAreAllowed = target != KonanTarget.WASM32
internal val produce get() = configuration.get(KonanConfigKeys.PRODUCE)!!
private val prefix = produce.prefix(target)
private val suffix = produce.suffix(target)
val outputName = configuration.get(KonanConfigKeys.OUTPUT)?.removeSuffixIfPresent(suffix) ?: produce.visibleName
val outputFile = outputName
.prefixBaseNameIfNot(prefix)
.suffixIfNot(suffix)
val tempFiles = TempFiles(outputName)
val outputFiles = OutputFiles(configuration.get(KonanConfigKeys.OUTPUT), target, produce)
val tempFiles = TempFiles(outputFiles.outputName, configuration.get(KonanConfigKeys.TEMPORARY_FILES_DIR))
val outputFile = outputFiles.mainFile
val moduleId: String
get() = configuration.get(KonanConfigKeys.MODULE_NAME) ?: File(outputName).name
get() = configuration.get(KonanConfigKeys.MODULE_NAME) ?: File(outputFiles.outputName).name
internal val purgeUserLibs: Boolean
get() = configuration.getBoolean(KonanConfigKeys.PURGE_USER_LIBS)
@@ -93,6 +93,8 @@ class KonanConfigKeys {
= CompilerConfigurationKey.create("generate source map")
val TARGET: CompilerConfigurationKey<String?>
= CompilerConfigurationKey.create("target we compile for")
val TEMPORARY_FILES_DIR: CompilerConfigurationKey<String?>
= CompilerConfigurationKey.create("directory for temporary files")
val TIME_PHASES: CompilerConfigurationKey<Boolean>
= CompilerConfigurationKey.create("time backend phases")
val VERIFY_BITCODE: CompilerConfigurationKey<Boolean>
@@ -68,11 +68,8 @@ internal class LinkStage(val context: Context) {
return combined
}
private fun temporary(name: String, suffix: String): String {
val temporaryFile = createTempFile(name, suffix)
temporaryFile.deleteOnExit()
return temporaryFile.absolutePath
}
private fun temporary(name: String, suffix: String): String =
context.config.tempFiles.create(name, suffix).absolutePath
private fun targetTool(tool: String, vararg arg: String) {
val absoluteToolName = "${platform.absoluteTargetToolchain}/bin/$tool"
@@ -0,0 +1,48 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan
import org.jetbrains.kotlin.backend.konan.util.prefixBaseNameIfNot
import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.visibleName
/**
* Creates and stores terminal compiler outputs
*/
class OutputFiles(outputPath: String?, target: KonanTarget, produce: CompilerOutputKind) {
private val prefix = produce.prefix(target)
private val suffix = produce.suffix(target)
val outputName = outputPath?.removeSuffixIfPresent(suffix) ?: produce.visibleName
/**
* Header file for dynamic library
*/
val cAdapterHeader by lazy { File("${outputName}_api.h") }
/**
* Main compiler's output file
*/
val mainFile = outputName
.prefixBaseNameIfNot(prefix)
.suffixIfNot(suffix)
}
@@ -1,32 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.*
class TempFiles(val outputName: String) {
val nativeBinaryFile by lazy { File("${outputName}.kt.bc") }
val cAdapterHeader by lazy { File("${outputName}_api.h") }
val cAdapterDef by lazy { File("${outputName}_symbols.def") }
val cAdapterCpp by lazy { createTempFile("api", ".cpp").deleteOnExit() }
val cAdapterBitcode by lazy { createTempFile("api", ".bc").deleteOnExit() }
val nativeBinaryFileName get() = nativeBinaryFile.absolutePath
val cAdapterCppName get() = cAdapterCpp.absolutePath
val cAdapterBitcodeName get() = cAdapterBitcode.absolutePath
}
@@ -154,6 +154,10 @@ class NamedNativeInteropConfig implements Named {
return new File(project.buildDir, "nativeInteropStubs/$name/kotlin")
}
File getTemporaryFilesDir() {
return new File(project.buildDir, "interopTemp")
}
NamedNativeInteropConfig(Project project, String name, String target = null, String flavor = 'jvm') {
this.name = name
this.project = project
@@ -204,6 +208,7 @@ class NamedNativeInteropConfig implements Named {
outputs.dir generatedSrcDir
outputs.dir nativeLibsDir
outputs.dir temporaryFilesDir
// defer as much as possible
doFirst {
@@ -217,6 +222,7 @@ class NamedNativeInteropConfig implements Named {
args '-generated', generatedSrcDir
args '-natives', nativeLibsDir
args '-temporaryFilesDir', temporaryFilesDir
args '-flavor', this.flavor
// Uncomment to debug.
// args '-verbose', 'true'
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.*
/**
* Creates and stores temporary compiler outputs
* If pathToTemporaryDir is given and is not empty then temporary outputs will be preserved
*/
class TempFiles(outputPath: String, pathToTemporaryDir: String? = null) {
private val outputName = File(outputPath).name
val nativeBinaryFile by lazy { File(dir,"${outputName}.kt.bc") }
val cAdapterDef by lazy { File(dir,"${outputName}_symbols.def") }
val cAdapterCpp by lazy { File(dir, "api.cpp") }
val cAdapterBitcode by lazy { File(dir, "api.bc") }
val nativeBinaryFileName get() = nativeBinaryFile.absolutePath
val cAdapterCppName get() = cAdapterCpp.absolutePath
val cAdapterBitcodeName get() = cAdapterBitcode.absolutePath
private val dir by lazy { if (pathToTemporaryDir == null || pathToTemporaryDir.isEmpty()) {
createTempDir("konan_temp").deleteOnExit()
} else {
createDirForTemporaryFiles(pathToTemporaryDir)
}
}
private fun createDirForTemporaryFiles(path: String): File {
if (File(path).isFile) {
throw IllegalArgumentException("Given file is not a directory: $path")
}
return File(path).apply {
if (!exists) { mkdirs() }
}
}
/**
* Create file named {name}{suffix} inside temporary dir
*/
fun create(prefix: String, suffix: String = ""): File =
File(dir, "$prefix$suffix")
}
@@ -66,7 +66,10 @@ open class Command(initialCommand: List<String>) {
handleExitCode(code)
}
fun getOutputLines(): List<String> {
/**
* If withErrors is true then output from error stream will be added
*/
fun getOutputLines(withErrors: Boolean = false): List<String> {
log()
val outputFile = createTempFile()
@@ -78,6 +81,7 @@ open class Command(initialCommand: List<String>) {
builder.redirectInput(Redirect.INHERIT)
builder.redirectError(Redirect.INHERIT)
builder.redirectOutput(ProcessBuilder.Redirect.to(outputFile))
.redirectErrorStream(withErrors)
// Note: getting process output could be done without redirecting to temporary file,
// however this would require managing a thread to read `process.inputStream` because
// it may have limited capacity.
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.backend.konan.library.defaultResolver
import org.jetbrains.kotlin.backend.konan.library.impl.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
import org.jetbrains.kotlin.konan.TempFiles
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.target.PlatformManager
@@ -40,6 +41,7 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
val repos = mutableListOf<String>()
var noDefaultLibs = false
var purgeUserLibs = false
var temporaryFilesDir = ""
for (i in args.indices) {
val arg = args[i]
val nextArg = args.getOrNull(i + 1)
@@ -55,6 +57,8 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
noDefaultLibs = true
if (arg == PURGE_USER_LIBS)
purgeUserLibs = true
if (arg == "--temporary_files_dir")
temporaryFilesDir = nextArg ?: ""
}
@@ -82,17 +86,18 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
} ?: emptyList()
}
val additionalArgs = listOf<String>(
val additionalArgs = listOf(
"-generated", generatedDir.path,
"-natives", nativesDir.path,
"-cstubsname", cstubsName,
"-manifest", manifest.path,
"-flavor", flavor
"-flavor", flavor,
"-temporaryFilesDir", temporaryFilesDir
) + importArgs
val cinteropArgs = (additionalArgs + args.filter { it !in cinteropArgFilter }).toTypedArray()
val cinteropArgsToCompiler = interop(flavor, cinteropArgs) ?: emptyArray<String>()
val cinteropArgsToCompiler = interop(flavor, cinteropArgs) ?: emptyArray()
val nativeStubs =
if (flavor == "wasm")
@@ -105,7 +110,8 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
"-produce", "library",
"-o", outputFileName,
"-target", target.visibleName,
"-manifest", manifest.path) +
"-manifest", manifest.path,
"--temporary_files_dir", temporaryFilesDir) +
nativeStubs +
cinteropArgsToCompiler +
libraries.flatMap { listOf("-library", it) } +