Use Workers API for NMPP tasks

This change enables parallel execution of compile tasks in NMPP projects
within a subproject.

    #KT-28155 In Progress
This commit is contained in:
Alexey Tsvetkov
2018-10-25 00:30:15 +03:00
parent ba5795519b
commit 37dfe2b608
22 changed files with 700 additions and 468 deletions
@@ -0,0 +1,12 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
object KotlinCompilerClass {
const val JVM = "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler"
const val JS = "org.jetbrains.kotlin.cli.js.K2JSCompiler"
const val METADATA = "org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler"
}
@@ -43,115 +43,28 @@ interface KotlinLogger {
}
abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
protected val K2JVM_COMPILER = "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler"
protected val K2JS_COMPILER = "org.jetbrains.kotlin.cli.js.K2JSCompiler"
protected val K2METADATA_COMPILER = "org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler"
protected val INTERNAL_ERROR = ExitCode.INTERNAL_ERROR.toString()
protected abstract val log: KotlinLogger
protected abstract fun getDaemonConnection(environment: Env): CompileServiceSession?
@Synchronized
protected fun newDaemonConnection(
compilerId: CompilerId,
clientAliveFlagFile: File,
sessionAliveFlagFile: File,
environment: Env,
daemonOptions: DaemonOptions = configureDaemonOptions(),
additionalJvmParams: Array<String> = arrayOf()
): CompileServiceSession? {
val daemonJVMOptions = configureDaemonJVMOptions(
additionalParams = *additionalJvmParams,
inheritMemoryLimits = true,
inheritOtherJvmOptions = false,
inheritAdditionalProperties = true
)
val daemonReportMessages = ArrayList<DaemonReportMessage>()
val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)
val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler()
val connection = profiler.withMeasure(null) {
KotlinCompilerClient.connectAndLease(compilerId,
clientAliveFlagFile,
daemonJVMOptions,
daemonOptions,
daemonReportingTargets,
autostart = true,
leaseSession = true,
sessionAliveFlagFile = sessionAliveFlagFile)
}
if (connection == null || log.isDebugEnabled) {
for (message in daemonReportMessages) {
val severity = when(message.category) {
DaemonReportCategory.DEBUG -> CompilerMessageSeverity.INFO
DaemonReportCategory.INFO -> CompilerMessageSeverity.INFO
DaemonReportCategory.EXCEPTION -> CompilerMessageSeverity.EXCEPTION
}
environment.messageCollector.report(severity, message.message)
}
}
fun reportTotalAndThreadPerf(message: String, daemonOptions: DaemonOptions, messageCollector: MessageCollector, profiler: Profiler) {
if (daemonOptions.reportPerf) {
fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this)
val counters = profiler.getTotalCounters()
messageCollector.report(CompilerMessageSeverity.INFO, "PERF: $message ${counters.time.ms()} ms, thread ${counters.threadTime.ms()}")
}
}
reportTotalAndThreadPerf("Daemon connect", daemonOptions, environment.messageCollector, profiler)
return connection
}
protected fun processCompilerOutput(
environment: Env,
stream: ByteArrayOutputStream,
exitCode: ExitCode?
protected open fun runCompiler(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: Env
) {
val reader = BufferedReader(StringReader(stream.toString()))
CompilerOutputParser.parseCompilerMessagesFromReader(environment.messageCollector, reader, environment.outputItemsCollector)
if (ExitCode.INTERNAL_ERROR == exitCode) {
reportInternalCompilerError(environment.messageCollector)
}
}
protected fun reportInternalCompilerError(messageCollector: MessageCollector): ExitCode {
messageCollector.report(CompilerMessageSeverity.ERROR, "Compiler terminated with internal error")
return ExitCode.INTERNAL_ERROR
}
protected fun runCompiler(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: Env): ExitCode {
return try {
try {
compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment)
}
catch (e: Throwable) {
} catch (e: Throwable) {
MessageCollectorUtil.reportException(environment.messageCollector, e)
reportInternalCompilerError(environment.messageCollector)
}
}
protected abstract fun compileWithDaemonOrFallback(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: Env
): ExitCode
/**
* Returns null if could not connect to daemon
*/
protected abstract fun compileWithDaemon(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: Env
): ExitCode?
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: Env
)
protected fun exitCodeFromProcessExitCode(code: Int): ExitCode = Companion.exitCodeFromProcessExitCode(log, code)
@@ -163,6 +76,69 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
log.debug("Could not find exit code by value: $code")
return if (code == 0) ExitCode.OK else ExitCode.COMPILATION_ERROR
}
@Synchronized
@JvmStatic
protected fun newDaemonConnection(
compilerId: CompilerId,
clientAliveFlagFile: File,
sessionAliveFlagFile: File,
messageCollector: MessageCollector,
isDebugEnabled: Boolean,
daemonOptions: DaemonOptions = configureDaemonOptions(),
additionalJvmParams: Array<String> = arrayOf()
): CompileServiceSession? {
val daemonJVMOptions = configureDaemonJVMOptions(
additionalParams = *additionalJvmParams,
inheritMemoryLimits = true,
inheritOtherJvmOptions = false,
inheritAdditionalProperties = true
)
val daemonReportMessages = ArrayList<DaemonReportMessage>()
val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)
val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler()
val connection = profiler.withMeasure(null) {
KotlinCompilerClient.connectAndLease(
compilerId,
clientAliveFlagFile,
daemonJVMOptions,
daemonOptions,
daemonReportingTargets,
autostart = true,
leaseSession = true,
sessionAliveFlagFile = sessionAliveFlagFile
)
}
if (connection == null || isDebugEnabled) {
for (message in daemonReportMessages) {
val severity = when (message.category) {
DaemonReportCategory.DEBUG -> CompilerMessageSeverity.INFO
DaemonReportCategory.INFO -> CompilerMessageSeverity.INFO
DaemonReportCategory.EXCEPTION -> CompilerMessageSeverity.EXCEPTION
}
messageCollector.report(severity, message.message)
}
}
fun reportTotalAndThreadPerf(
message: String, daemonOptions: DaemonOptions, messageCollector: MessageCollector, profiler: Profiler
) {
if (daemonOptions.reportPerf) {
fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this)
val counters = profiler.getTotalCounters()
messageCollector.report(
CompilerMessageSeverity.INFO, "PERF: $message ${counters.time.ms()} ms, thread ${counters.threadTime.ms()}"
)
}
}
reportTotalAndThreadPerf("Daemon connect", daemonOptions, MessageCollector.NONE, profiler)
return connection
}
}
}
@@ -0,0 +1,39 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import java.io.BufferedReader
import java.io.ByteArrayOutputStream
import java.io.StringReader
fun reportInternalCompilerError(messageCollector: MessageCollector) {
messageCollector.report(CompilerMessageSeverity.ERROR, "Compiler terminated with internal error")
}
fun processCompilerOutput(
environment: CompilerEnvironment,
stream: ByteArrayOutputStream,
exitCode: ExitCode?
) {
processCompilerOutput(environment.messageCollector, environment.outputItemsCollector, stream, exitCode)
}
fun processCompilerOutput(
messageCollector: MessageCollector,
outputItemsCollector: OutputItemsCollector,
stream: ByteArrayOutputStream,
exitCode: ExitCode?
) {
val reader = BufferedReader(StringReader(stream.toString()))
CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, outputItemsCollector)
if (ExitCode.INTERNAL_ERROR == exitCode) {
reportInternalCompilerError(messageCollector)
}
}
@@ -17,8 +17,13 @@
package org.jetbrains.kotlin.incremental
import java.io.File
import java.io.Serializable
sealed class ChangedFiles {
sealed class ChangedFiles : Serializable {
class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
class Unknown : ChangedFiles()
companion object {
const val serialVersionUID: Long = 0
}
}
@@ -105,7 +105,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
arguments.destination = arguments.destination ?: destination
withCompilerSettings(compilerSettings) {
runCompiler(K2METADATA_COMPILER, arguments, environment)
runCompiler(KotlinCompilerClass.METADATA, arguments, environment)
}
}
@@ -119,7 +119,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2jvmArguments))
setupK2JvmArguments(moduleFile, arguments)
withCompilerSettings(compilerSettings) {
runCompiler(K2JVM_COMPILER, arguments, environment)
runCompiler(KotlinCompilerClass.JVM, arguments, environment)
}
}
@@ -149,7 +149,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
log.debug("K2JS: arguments after setup" + ArgumentUtils.convertArgumentsToStringList(arguments))
withCompilerSettings(compilerSettings) {
runCompiler(K2JS_COMPILER, arguments, environment)
runCompiler(KotlinCompilerClass.JS, arguments, environment)
}
}
@@ -157,24 +157,24 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
): ExitCode {
) {
log.debug("Using kotlin-home = " + environment.kotlinPaths.homePath)
return withDaemonOrFallback(
withDaemonOrFallback(
withDaemon = { compileWithDaemon(compilerClassName, compilerArgs, environment) },
fallback = { fallbackCompileStrategy(compilerArgs, compilerClassName, environment) }
)
}
override fun compileWithDaemon(
private fun compileWithDaemon(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: JpsCompilerEnvironment
): ExitCode? {
) {
val targetPlatform = when (compilerClassName) {
K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
K2JS_COMPILER -> CompileService.TargetPlatform.JS
K2METADATA_COMPILER -> CompileService.TargetPlatform.METADATA
KotlinCompilerClass.JVM -> CompileService.TargetPlatform.JVM
KotlinCompilerClass.JS -> CompileService.TargetPlatform.JS
KotlinCompilerClass.METADATA -> CompileService.TargetPlatform.METADATA
else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
}
val compilerMode = CompilerMode.JPS_COMPILER
@@ -186,7 +186,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
reportSeverity(verbose),
requestedCompilationResults = emptyArray()
)
return doWithDaemon(environment) { sessionId, daemon ->
doWithDaemon(environment) { sessionId, daemon ->
environment.withProgressReporter { progress ->
progress.compilationStarted()
daemon.compile(
@@ -197,7 +197,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
null
)
}
}?.let { exitCodeFromProcessExitCode(it) }
}
}
private fun <T> withDaemonOrFallback(withDaemon: () -> T?, fallback: () -> T): T =
@@ -253,7 +253,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
compilerArgs: CommonCompilerArguments,
compilerClassName: String,
environment: JpsCompilerEnvironment
): ExitCode {
) {
if ("true" == System.getProperty("kotlin.jps.tests") && "true" == System.getProperty(FAIL_ON_FALLBACK_PROPERTY)) {
error("Fallback strategy is disabled in tests!")
}
@@ -278,8 +278,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
// exec() returns an ExitCode object, class of which is loaded with a different class loader,
// so we take it's contents through reflection
val exitCode = ExitCode.valueOf(getReturnCodeFromObject(rc))
processCompilerOutput(environment, stream, exitCode)
return exitCode
processCompilerOutput(environment.messageCollector, environment.outputItemsCollector, stream, exitCode)
}
private fun setupK2JvmArguments(moduleFile: File, settings: K2JVMCompilerArguments) {
@@ -317,7 +316,7 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
else -> throw IllegalStateException("Unexpected return: " + rc)
}
override fun getDaemonConnection(environment: JpsCompilerEnvironment): CompileServiceSession? =
private fun getDaemonConnection(environment: JpsCompilerEnvironment): CompileServiceSession? =
getOrCreateDaemonConnection {
environment.progressReporter.progress("connecting to daemon")
val libPath = CompilerRunnerUtil.getLibPath(environment.kotlinPaths, environment.messageCollector)
@@ -334,7 +333,15 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
environment.withProgressReporter { progress ->
progress.progress("connecting to daemon")
newDaemonConnection(compilerId, clientFlagFile, sessionFlagFile, environment, daemonOptions, additionalJvmParams.toTypedArray())
newDaemonConnection(
compilerId,
clientFlagFile,
sessionFlagFile,
environment.messageCollector,
log.isDebugEnabled,
daemonOptions,
additionalJvmParams.toTypedArray()
)
}
}
}
@@ -1,6 +1,5 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
import org.jetbrains.kotlin.daemon.common.CompilationResults
import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface
@@ -8,22 +7,20 @@ import org.jetbrains.kotlin.daemon.common.SOCKET_ANY_FREE_PORT
import org.jetbrains.kotlin.daemon.common.CompileIterationResult
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.utils.pathsAsStringRelativeTo
import java.io.File
import java.io.Serializable
import java.rmi.RemoteException
import java.rmi.server.UnicastRemoteObject
internal class GradleCompilationResults(
project: Project
private val log: KotlinLogger,
private val projectRootFile: File
) : CompilationResults,
UnicastRemoteObject(
SOCKET_ANY_FREE_PORT,
LoopbackNetworkInterface.clientLoopbackSocketFactory,
LoopbackNetworkInterface.serverLoopbackSocketFactory
) {
private val log = project.logger
private val projectRootFile = project.rootProject.projectDir
@Throws(RemoteException::class)
override fun add(compilationResultCategory: Int, value: Serializable) {
if (compilationResultCategory == CompilationResultCategory.IC_COMPILE_ITERATION.code) {
@@ -5,38 +5,20 @@
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
import org.jetbrains.kotlin.gradle.tasks.findToolsJar
import org.jetbrains.kotlin.incremental.ChangedFiles
import java.io.File
import java.net.URL
internal open class GradleCompilerEnvironment(
internal class GradleCompilerEnvironment(
val compilerClasspath: List<File>,
messageCollector: GradleMessageCollector,
outputItemsCollector: OutputItemsCollector,
val compilerArgs: CommonCompilerArguments
val incrementalCompilationEnvironment: IncrementalCompilationEnvironment? = null
) : CompilerEnvironment(Services.EMPTY, messageCollector, outputItemsCollector) {
val toolsJar: File? by lazy { findToolsJar() }
val compilerFullClasspath: List<File>
get() = (compilerClasspath + toolsJar).filterNotNull()
val compilerClasspathURLs: List<URL>
get() = compilerFullClasspath.map { it.toURI().toURL() }
}
internal class GradleIncrementalCompilerEnvironment(
compilerClasspath: List<File>,
val changedFiles: ChangedFiles,
val workingDir: File,
messageCollector: GradleMessageCollector,
outputItemsCollector: OutputItemsCollector,
compilerArgs: CommonCompilerArguments,
val usePreciseJavaTracking: Boolean = false,
val localStateDirs: List<File> = emptyList(),
val multiModuleICSettings: MultiModuleICSettings
) : GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemsCollector, compilerArgs)
@@ -0,0 +1,36 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.workers.IsolationMode
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
internal class GradleCompilerRunnerWithWorkers(
project: Project,
private val workersExecutor: WorkerExecutor
) : GradleCompilerRunner(project) {
override fun compileWithDaemonOrFallback(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: GradleCompilerEnvironment
) {
workersExecutor.submit(KotlinCompilerRunnable::class.java) { config ->
config.isolationMode = IsolationMode.NONE
config.params(
ProjectFilesForCompilation(project),
environment.compilerFullClasspath,
compilerClassName,
ArgumentUtils.convertArgumentsToStringList(compilerArgs).toTypedArray(),
compilerArgs.verbose,
environment.incrementalCompilationEnvironment,
buildModulesInfo(project.gradle)
)
}
}
}
@@ -5,8 +5,6 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.daemon.client.reportFromDaemon
import org.jetbrains.kotlin.daemon.common.*
@@ -16,15 +14,13 @@ import java.rmi.Remote
import java.rmi.server.UnicastRemoteObject
internal open class GradleCompilerServicesFacadeImpl(
project: Project,
val compilerMessageCollector: MessageCollector,
private val log: KotlinLogger,
private val compilerMessageCollector: MessageCollector,
port: Int = SOCKET_ANY_FREE_PORT
) : UnicastRemoteObject(port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory),
CompilerServicesFacadeBase,
Remote {
protected val log: Logger = project.logger
override fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) {
when (ReportCategory.fromCode(category)) {
ReportCategory.IC_MESSAGE -> {
@@ -47,8 +43,8 @@ internal open class GradleCompilerServicesFacadeImpl(
}
internal class GradleIncrementalCompilerServicesFacadeImpl(
project: Project,
environment: GradleIncrementalCompilerEnvironment,
log: KotlinLogger,
messageCollector: MessageCollector,
port: Int = SOCKET_ANY_FREE_PORT
) : GradleCompilerServicesFacadeImpl(project, environment.messageCollector, port),
) : GradleCompilerServicesFacadeImpl(log, messageCollector, port),
IncrementalCompilerServicesFacade
@@ -21,31 +21,20 @@ import org.gradle.api.invocation.Gradle
import org.gradle.api.plugins.JavaPluginConvention
import org.gradle.jvm.tasks.Jar
import org.jetbrains.kotlin.build.JvmSourceRoot
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.client.CompileServiceSession
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
import org.jetbrains.kotlin.gradle.tasks.InspectClassesForMultiModuleIC
import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
import org.jetbrains.kotlin.gradle.tasks.*
import org.jetbrains.kotlin.gradle.utils.newTmpFile
import org.jetbrains.kotlin.gradle.utils.relativeToRoot
import org.jetbrains.kotlin.incremental.*
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.PrintStream
import java.lang.ref.WeakReference
import java.net.URLClassLoader
import java.rmi.RemoteException
internal const val KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY = "kotlin.compiler.execution.strategy"
internal const val DAEMON_EXECUTION_STRATEGY = "daemon"
@@ -58,37 +47,12 @@ const val EXISTING_SESSION_FILE_PREFIX = "Existing session-is-alive flag file: "
const val DELETED_SESSION_FILE_PREFIX = "Deleted session-is-alive flag file: "
const val COULD_NOT_CONNECT_TO_DAEMON_MESSAGE = "Could not connect to Kotlin compile daemon"
internal class GradleCompilerRunner(private val project: Project) : KotlinCompilerRunner<GradleCompilerEnvironment>() {
internal fun kotlinCompilerExecutionStrategy(): String =
System.getProperty(KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY) ?: DAEMON_EXECUTION_STRATEGY
internal open class GradleCompilerRunner(protected val project: Project) : KotlinCompilerRunner<GradleCompilerEnvironment>() {
override val log = GradleKotlinLogger(project.logger)
// used only for process launching so far, but implements unused proper contract
private val loggingMessageCollector: MessageCollector by lazy {
object : MessageCollector {
private var hasErrors = false
private val messageRenderer = MessageRenderer.PLAIN_FULL_PATHS
override fun clear() {
hasErrors = false
}
override fun hasErrors(): Boolean = hasErrors
override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
val locMessage = messageRenderer.render(severity, message, location)
when (severity) {
CompilerMessageSeverity.EXCEPTION -> log.error(locMessage)
CompilerMessageSeverity.ERROR,
CompilerMessageSeverity.STRONG_WARNING,
CompilerMessageSeverity.WARNING,
CompilerMessageSeverity.INFO -> log.info(locMessage)
CompilerMessageSeverity.LOGGING -> log.debug(locMessage)
CompilerMessageSeverity.OUTPUT -> {
}
}
}
}
}
fun runJvmCompiler(
sourcesToCompile: List<File>,
commonSources: List<File>,
@@ -96,7 +60,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
javaPackagePrefix: String?,
args: K2JVMCompilerArguments,
environment: GradleCompilerEnvironment
): ExitCode {
) {
val buildFile = makeModuleFile(
args.moduleName!!,
isTest = false,
@@ -109,12 +73,12 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
)
args.buildFile = buildFile.absolutePath
if (environment !is GradleIncrementalCompilerEnvironment || kotlinCompilerExecutionStrategy != "daemon") {
if (environment.incrementalCompilationEnvironment == null || kotlinCompilerExecutionStrategy() != DAEMON_EXECUTION_STRATEGY) {
args.destination = null
}
try {
return runCompiler(K2JVM_COMPILER, args, environment)
runCompiler(KotlinCompilerClass.JVM, args, environment)
} finally {
if (System.getProperty(DELETE_MODULE_FILE_PROPERTY) != "false") {
buildFile.delete()
@@ -127,26 +91,26 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
kotlinCommonSources: List<File>,
args: K2JSCompilerArguments,
environment: GradleCompilerEnvironment
): ExitCode {
) {
args.freeArgs += kotlinSources.map { it.absolutePath }
args.commonSources = kotlinCommonSources.map { it.absolutePath }.toTypedArray()
return runCompiler(K2JS_COMPILER, args, environment)
runCompiler(KotlinCompilerClass.JS, args, environment)
}
fun runMetadataCompiler(
kotlinSources: List<File>,
args: K2MetadataCompilerArguments,
environment: GradleCompilerEnvironment
): ExitCode {
) {
args.freeArgs += kotlinSources.map { it.absolutePath }
return runCompiler(K2METADATA_COMPILER, args, environment)
return runCompiler(KotlinCompilerClass.METADATA, args, environment)
}
override fun compileWithDaemonOrFallback(
override fun runCompiler(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: GradleCompilerEnvironment
): ExitCode {
) {
if (compilerArgs.version) {
project.logger.lifecycle(
"Kotlin version " + loadCompilerVersion(environment.compilerClasspath) +
@@ -154,206 +118,50 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
)
compilerArgs.version = false
}
val argsArray = ArgumentUtils.convertArgumentsToStringList(compilerArgs).toTypedArray()
with(project.logger) {
kotlinDebug { "Kotlin compiler class: $compilerClassName" }
kotlinDebug { "Kotlin compiler classpath: ${environment.compilerFullClasspath.map { it.canonicalPath }.joinToString()}" }
kotlinDebug { "Kotlin compiler args: ${argsArray.joinToString(" ")}" }
}
val executionStrategy = kotlinCompilerExecutionStrategy
if (executionStrategy == DAEMON_EXECUTION_STRATEGY) {
val daemonExitCode = compileWithDaemon(compilerClassName, compilerArgs, environment)
if (daemonExitCode != null) {
return daemonExitCode
} else {
log.warn("Could not connect to kotlin daemon. Using fallback strategy.")
}
}
val isGradleDaemonUsed = System.getProperty("org.gradle.daemon")?.let(String::toBoolean)
return if (executionStrategy == IN_PROCESS_EXECUTION_STRATEGY || isGradleDaemonUsed == false) {
compileInProcess(argsArray, compilerClassName, environment)
} else {
compileOutOfProcess(argsArray, compilerClassName, environment)
}
super.runCompiler(compilerClassName, compilerArgs, environment)
}
private val kotlinCompilerExecutionStrategy: String
get() = System.getProperty(KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY) ?: DAEMON_EXECUTION_STRATEGY
override fun compileWithDaemon(
override fun compileWithDaemonOrFallback(
compilerClassName: String,
compilerArgs: CommonCompilerArguments,
environment: GradleCompilerEnvironment
): ExitCode? {
val connection =
try {
getDaemonConnection(environment)
} catch (e: Throwable) {
log.warn("Caught an exception trying to connect to Kotlin Daemon")
e.printStackTrace()
null
}
if (connection == null) {
if (environment is GradleIncrementalCompilerEnvironment) {
log.warn("Could not perform incremental compilation: $COULD_NOT_CONNECT_TO_DAEMON_MESSAGE")
} else {
log.warn(COULD_NOT_CONNECT_TO_DAEMON_MESSAGE)
}
return null
}
val (daemon, sessionId) = connection
val targetPlatform = when (compilerClassName) {
K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
K2JS_COMPILER -> CompileService.TargetPlatform.JS
K2METADATA_COMPILER -> CompileService.TargetPlatform.METADATA
else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
}
val exitCode = try {
val res = if (environment is GradleIncrementalCompilerEnvironment) {
incrementalCompilationWithDaemon(daemon, sessionId, targetPlatform, environment)
} else {
nonIncrementalCompilationWithDaemon(daemon, sessionId, targetPlatform, environment)
}
exitCodeFromProcessExitCode(res.get())
} catch (e: Throwable) {
log.warn("Compilation with Kotlin compile daemon was not successful")
e.printStackTrace()
null
}
// todo: can we clear cache on the end of session?
// often source of the NoSuchObjectException and UnmarshalException, probably caused by the failed/crashed/exited daemon
// TODO: implement a proper logic to avoid remote calls in such cases
try {
daemon.clearJarCache()
} catch (e: RemoteException) {
log.warn("Unable to clear jar cache after compilation, maybe daemon is already down: $e")
}
logFinish(DAEMON_EXECUTION_STRATEGY)
return exitCode
}
private fun nonIncrementalCompilationWithDaemon(
daemon: CompileService,
sessionId: Int,
targetPlatform: CompileService.TargetPlatform,
environment: GradleCompilerEnvironment
): CompileService.CallResult<Int> {
val verbose = environment.compilerArgs.verbose
val compilationOptions = CompilationOptions(
compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
targetPlatform = targetPlatform,
reportCategories = reportCategories(verbose),
reportSeverity = reportSeverity(verbose),
requestedCompilationResults = emptyArray()
) {
val kotlinCompilerRunnable = KotlinCompilerRunnable(
projectFiles = ProjectFilesForCompilation(project),
compilerFullClasspath = environment.compilerFullClasspath,
compilerClassName = compilerClassName,
compilerArgs = ArgumentUtils.convertArgumentsToStringList(compilerArgs).toTypedArray(),
isVerbose = compilerArgs.verbose,
incrementalCompilationEnvironment = environment.incrementalCompilationEnvironment,
incrementalModuleInfo = buildModulesInfo(project.gradle)
)
val servicesFacade = GradleCompilerServicesFacadeImpl(project, environment.messageCollector)
val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
return daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, compilationResults = null)
}
private fun incrementalCompilationWithDaemon(
daemon: CompileService,
sessionId: Int,
targetPlatform: CompileService.TargetPlatform,
environment: GradleIncrementalCompilerEnvironment
): CompileService.CallResult<Int> {
val knownChangedFiles = environment.changedFiles as? ChangedFiles.Known
val verbose = environment.compilerArgs.verbose
val compilationOptions = IncrementalCompilationOptions(
areFileChangesKnown = knownChangedFiles != null,
modifiedFiles = knownChangedFiles?.modified,
deletedFiles = knownChangedFiles?.removed,
workingDir = environment.workingDir,
reportCategories = reportCategories(verbose),
reportSeverity = reportSeverity(verbose),
requestedCompilationResults = arrayOf(CompilationResultCategory.IC_COMPILE_ITERATION.code),
compilerMode = CompilerMode.INCREMENTAL_COMPILER,
targetPlatform = targetPlatform,
usePreciseJavaTracking = environment.usePreciseJavaTracking,
localStateDirs = environment.localStateDirs,
multiModuleICSettings = environment.multiModuleICSettings,
modulesInfo = buildModulesInfo(project.gradle)
)
log.info("Options for KOTLIN DAEMON: $compilationOptions")
val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(project, environment)
val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
return daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, GradleCompilationResults(project))
}
private fun reportCategories(verbose: Boolean): Array<Int> =
if (!verbose) {
arrayOf(ReportCategory.COMPILER_MESSAGE.code)
} else {
ReportCategory.values().map { it.code }.toTypedArray()
}
private fun reportSeverity(verbose: Boolean): Int =
if (!verbose) {
ReportSeverity.INFO.code
} else {
ReportSeverity.DEBUG.code
}
private fun compileOutOfProcess(
argsArray: Array<String>,
compilerClassName: String,
environment: GradleCompilerEnvironment
): ExitCode {
return runToolInSeparateProcess(argsArray, compilerClassName, environment.compilerFullClasspath, log, loggingMessageCollector)
}
private fun compileInProcess(
argsArray: Array<String>,
compilerClassName: String,
environment: GradleCompilerEnvironment
): ExitCode {
val stream = ByteArrayOutputStream()
val out = PrintStream(stream)
// todo: cache classloader?
val classLoader = URLClassLoader(environment.compilerClasspathURLs.toTypedArray())
val servicesClass = Class.forName(Services::class.java.canonicalName, true, classLoader)
val emptyServices = servicesClass.getField("EMPTY").get(servicesClass)
val compiler = Class.forName(compilerClassName, true, classLoader)
val exec = compiler.getMethod(
"execAndOutputXml",
PrintStream::class.java,
servicesClass,
Array<String>::class.java
)
val res = exec.invoke(compiler.newInstance(), out, emptyServices, argsArray)
val exitCode = ExitCode.valueOf(res.toString())
processCompilerOutput(environment, stream, exitCode)
logFinish(IN_PROCESS_EXECUTION_STRATEGY)
return exitCode
}
private fun logFinish(strategy: String) = log.logFinish(strategy)
override fun getDaemonConnection(environment: GradleCompilerEnvironment): CompileServiceSession? {
synchronized(this.javaClass) {
val compilerId = CompilerId.makeCompilerId(environment.compilerFullClasspath)
val clientIsAliveFlagFile = getOrCreateClientFlagFile(project)
val sessionIsAliveFlagFile = getOrCreateSessionFlagFile(project)
return newDaemonConnection(compilerId, clientIsAliveFlagFile, sessionIsAliveFlagFile, environment)
}
kotlinCompilerRunnable.run()
}
companion object {
@Synchronized
internal fun getDaemonConnectionImpl(
clientIsAliveFlagFile: File,
sessionIsAliveFlagFile: File,
compilerFullClasspath: List<File>,
messageCollector: MessageCollector,
isDebugEnabled: Boolean
): CompileServiceSession? {
val compilerId = CompilerId.makeCompilerId(compilerFullClasspath)
return newDaemonConnection(
compilerId, clientIsAliveFlagFile, sessionIsAliveFlagFile,
messageCollector = messageCollector,
isDebugEnabled = isDebugEnabled
)
}
@Volatile
private var cachedGradle = WeakReference<Gradle>(null)
@Volatile
private var cachedModulesInfo: IncrementalModuleInfo? = null
@Synchronized
private fun buildModulesInfo(gradle: Gradle): IncrementalModuleInfo {
internal fun buildModulesInfo(gradle: Gradle): IncrementalModuleInfo {
if (cachedGradle.get() === gradle && cachedModulesInfo != null) return cachedModulesInfo!!
val dirToModule = HashMap<File, IncrementalModuleEntry>()
@@ -412,7 +220,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
private var clientIsAliveFlagFile: File? = null
@Synchronized
private fun getOrCreateClientFlagFile(project: Project): File {
internal fun getOrCreateClientFlagFile(project: Project): File {
val log = project.logger
if (clientIsAliveFlagFile == null || !clientIsAliveFlagFile!!.exists()) {
val projectName = project.rootProject.name.normalizeForFlagFile()
@@ -436,7 +244,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
// session files are deleted at org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices.buildFinished
@Synchronized
private fun getOrCreateSessionFlagFile(project: Project): File {
internal fun getOrCreateSessionFlagFile(project: Project): File {
val log = project.logger
if (sessionFlagFile == null || !sessionFlagFile!!.exists()) {
val sessionFilesDir = sessionsDir(project).apply { mkdirs() }
@@ -453,3 +261,4 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
File(File(project.rootProject.buildDir, "kotlin"), "sessions")
}
}
@@ -0,0 +1,23 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
import org.jetbrains.kotlin.incremental.ChangedFiles
import java.io.File
import java.io.Serializable
internal class IncrementalCompilationEnvironment(
val changedFiles: ChangedFiles,
val workingDir: File,
val usePreciseJavaTracking: Boolean = false,
val localStateDirs: List<File> = emptyList(),
val multiModuleICSettings: MultiModuleICSettings
) : Serializable {
companion object {
const val serialVersionUID: Long = 0
}
}
@@ -0,0 +1,265 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.slf4j.LoggerFactory
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.PrintStream
import java.io.Serializable
import java.net.URLClassLoader
import java.rmi.RemoteException
import javax.inject.Inject
internal class ProjectFilesForCompilation(
val projectRootFile: File,
val clientIsAliveFlagFile: File,
val sessionFlagFile: File
) : Serializable {
constructor(project: Project) : this(
projectRootFile = project.rootProject.projectDir,
clientIsAliveFlagFile = GradleCompilerRunner.getOrCreateClientFlagFile(project),
sessionFlagFile = GradleCompilerRunner.getOrCreateSessionFlagFile(project)
)
companion object {
const val serialVersionUID: Long = 0
}
}
internal class KotlinCompilerRunnable @Inject constructor(
private val projectFiles: ProjectFilesForCompilation,
private val compilerFullClasspath: List<File>,
private val compilerClassName: String,
private val compilerArgs: Array<String>,
private val isVerbose: Boolean,
private val incrementalCompilationEnvironment: IncrementalCompilationEnvironment?,
private val incrementalModuleInfo: IncrementalModuleInfo?
) : Runnable {
private val log: KotlinLogger =
SL4JKotlinLogger(LoggerFactory.getLogger("KotlinCompilerRunnable"))
private val messageCollector = GradleMessageCollector(log)
private val isIncremental: Boolean
get() = incrementalCompilationEnvironment != null
override fun run() {
val exitCode = compileWithDaemonOrFallbackImpl()
throwGradleExceptionIfError(exitCode)
}
private fun compileWithDaemonOrFallbackImpl(): ExitCode {
with(log) {
kotlinDebug { "Kotlin compiler class: $compilerClassName" }
kotlinDebug { "Kotlin compiler classpath: ${compilerFullClasspath.map { it.canonicalPath }.joinToString()}" }
kotlinDebug { "Kotlin compiler args: ${compilerArgs.joinToString(" ")}" }
}
val executionStrategy = kotlinCompilerExecutionStrategy()
if (executionStrategy == DAEMON_EXECUTION_STRATEGY) {
val daemonExitCode = compileWithDaemon()
if (daemonExitCode != null) {
return daemonExitCode
} else {
log.warn("Could not connect to kotlin daemon. Using fallback strategy.")
}
}
val isGradleDaemonUsed = System.getProperty("org.gradle.daemon")?.let(String::toBoolean)
return if (executionStrategy == IN_PROCESS_EXECUTION_STRATEGY || isGradleDaemonUsed == false) {
compileInProcess()
} else {
compileOutOfProcess()
}
}
private fun compileWithDaemon(): ExitCode? {
val connection =
try {
GradleCompilerRunner.getDaemonConnectionImpl(
projectFiles.clientIsAliveFlagFile,
projectFiles.sessionFlagFile,
compilerFullClasspath,
messageCollector,
log.isDebugEnabled
)
} catch (e: Throwable) {
log.warn("Caught an exception trying to connect to Kotlin Daemon")
e.printStackTrace()
null
}
if (connection == null) {
if (isIncremental) {
log.warn("Could not perform incremental compilation: $COULD_NOT_CONNECT_TO_DAEMON_MESSAGE")
} else {
log.warn(COULD_NOT_CONNECT_TO_DAEMON_MESSAGE)
}
return null
}
val (daemon, sessionId) = connection
val targetPlatform = when (compilerClassName) {
KotlinCompilerClass.JVM -> CompileService.TargetPlatform.JVM
KotlinCompilerClass.JS -> CompileService.TargetPlatform.JS
KotlinCompilerClass.METADATA -> CompileService.TargetPlatform.METADATA
else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
}
val exitCode = try {
val res = if (isIncremental) {
incrementalCompilationWithDaemon(daemon, sessionId, targetPlatform)
} else {
nonIncrementalCompilationWithDaemon(daemon, sessionId, targetPlatform)
}
exitCodeFromProcessExitCode(log, res.get())
} catch (e: Throwable) {
log.warn("Compilation with Kotlin compile daemon was not successful")
e.printStackTrace()
null
}
// todo: can we clear cache on the end of session?
// often source of the NoSuchObjectException and UnmarshalException, probably caused by the failed/crashed/exited daemon
// TODO: implement a proper logic to avoid remote calls in such cases
try {
daemon.clearJarCache()
} catch (e: RemoteException) {
log.warn("Unable to clear jar cache after compilation, maybe daemon is already down: $e")
}
log.logFinish(DAEMON_EXECUTION_STRATEGY)
return exitCode
}
private fun nonIncrementalCompilationWithDaemon(
daemon: CompileService,
sessionId: Int,
targetPlatform: CompileService.TargetPlatform
): CompileService.CallResult<Int> {
val compilationOptions = CompilationOptions(
compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
targetPlatform = targetPlatform,
reportCategories = reportCategories(isVerbose),
reportSeverity = reportSeverity(isVerbose),
requestedCompilationResults = emptyArray()
)
val servicesFacade = GradleCompilerServicesFacadeImpl(log, messageCollector)
return daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults = null)
}
private fun incrementalCompilationWithDaemon(
daemon: CompileService,
sessionId: Int,
targetPlatform: CompileService.TargetPlatform
): CompileService.CallResult<Int> {
val icEnv = incrementalCompilationEnvironment ?: error("incrementalCompilationEnvironment is null!")
val knownChangedFiles = icEnv.changedFiles as? ChangedFiles.Known
val compilationOptions = IncrementalCompilationOptions(
areFileChangesKnown = knownChangedFiles != null,
modifiedFiles = knownChangedFiles?.modified,
deletedFiles = knownChangedFiles?.removed,
workingDir = icEnv.workingDir,
reportCategories = reportCategories(isVerbose),
reportSeverity = reportSeverity(isVerbose),
requestedCompilationResults = arrayOf(CompilationResultCategory.IC_COMPILE_ITERATION.code),
compilerMode = CompilerMode.INCREMENTAL_COMPILER,
targetPlatform = targetPlatform,
usePreciseJavaTracking = icEnv.usePreciseJavaTracking,
localStateDirs = icEnv.localStateDirs,
multiModuleICSettings = icEnv.multiModuleICSettings,
modulesInfo = incrementalModuleInfo!!
)
log.info("Options for KOTLIN DAEMON: $compilationOptions")
val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, messageCollector)
val compilationResults = GradleCompilationResults(log, projectFiles.projectRootFile)
return daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults)
}
private fun compileOutOfProcess(): ExitCode =
runToolInSeparateProcess(compilerArgs, compilerClassName, compilerFullClasspath, log, loggingMessageCollector)
private fun compileInProcess(): ExitCode {
val stream = ByteArrayOutputStream()
val out = PrintStream(stream)
// todo: cache classloader?
val classLoader = URLClassLoader(compilerFullClasspath.map { it.toURI().toURL() }.toTypedArray())
val servicesClass = Class.forName(Services::class.java.canonicalName, true, classLoader)
val emptyServices = servicesClass.getField("EMPTY").get(servicesClass)
val compiler = Class.forName(compilerClassName, true, classLoader)
val exec = compiler.getMethod(
"execAndOutputXml",
PrintStream::class.java,
servicesClass,
Array<String>::class.java
)
val res = exec.invoke(compiler.newInstance(), out, emptyServices, compilerArgs)
val exitCode = ExitCode.valueOf(res.toString())
processCompilerOutput(
messageCollector,
OutputItemsCollectorImpl(),
stream,
exitCode
)
log.logFinish(IN_PROCESS_EXECUTION_STRATEGY)
return exitCode
}
private fun reportCategories(verbose: Boolean): Array<Int> =
if (!verbose) {
arrayOf(ReportCategory.COMPILER_MESSAGE.code)
} else {
ReportCategory.values().map { it.code }.toTypedArray()
}
private fun reportSeverity(verbose: Boolean): Int =
if (!verbose) {
ReportSeverity.INFO.code
} else {
ReportSeverity.DEBUG.code
}
// used only for process launching so far, but implements unused proper contract
private val loggingMessageCollector: MessageCollector by lazy {
object : MessageCollector {
private var hasErrors = false
private val messageRenderer = MessageRenderer.PLAIN_FULL_PATHS
override fun clear() {
hasErrors = false
}
override fun hasErrors(): Boolean = hasErrors
override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
val locMessage = messageRenderer.render(severity, message, location)
when (severity) {
CompilerMessageSeverity.EXCEPTION -> log.error(locMessage)
CompilerMessageSeverity.ERROR,
CompilerMessageSeverity.STRONG_WARNING,
CompilerMessageSeverity.WARNING,
CompilerMessageSeverity.INFO -> log.info(locMessage)
CompilerMessageSeverity.LOGGING -> log.debug(locMessage)
CompilerMessageSeverity.OUTPUT -> {
}
}
}
}
}
}
@@ -0,0 +1,29 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.compilerRunner
import org.slf4j.Logger
internal class SL4JKotlinLogger(private val log: Logger) : KotlinLogger {
override fun debug(msg: String) {
log.debug(msg)
}
override fun error(msg: String) {
log.error(msg)
}
override fun info(msg: String) {
log.info(msg)
}
override fun warn(msg: String) {
log.warn(msg)
}
override val isDebugEnabled: Boolean
get() = log.isDebugEnabled
}
@@ -68,8 +68,11 @@ internal fun loadCompilerVersion(compilerClasspath: List<File>): String {
}
internal fun runToolInSeparateProcess(
argsArray: Array<String>, compilerClassName: String, classpath: List<File>,
logger: KotlinLogger, messageCollector: MessageCollector
argsArray: Array<String>,
compilerClassName: String,
classpath: List<File>,
logger: KotlinLogger,
messageCollector: MessageCollector
): ExitCode {
val javaBin = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"
val classpathString = classpath.map { it.absolutePath }.joinToString(separator = File.pathSeparator)
@@ -15,6 +15,7 @@ import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
import org.jetbrains.kotlin.compilerRunner.GradleKotlinLogger
import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
import org.jetbrains.kotlin.gradle.plugin.PLUGIN_CLASSPATH_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.tasks.CompilerPluginOptions
@@ -61,15 +62,15 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
val args = prepareCompilerArguments()
val messageCollector = GradleMessageCollector(logger)
val messageCollector = GradleMessageCollector(GradleKotlinLogger(logger))
val outputItemCollector = OutputItemsCollectorImpl()
val environment = GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemCollector, args)
val environment = GradleCompilerEnvironment(compilerClasspath, messageCollector, outputItemCollector)
if (environment.toolsJar == null && !isAtLeastJava9) {
throw GradleException("Could not find tools.jar in system classpath, which is required for kapt to work")
}
val compilerRunner = GradleCompilerRunner(project)
val exitCode = compilerRunner.runJvmCompiler(
compilerRunner.runJvmCompiler(
sourcesToCompile = emptyList(),
commonSources = emptyList(),
javaSourceRoots = javaSourceRoots,
@@ -77,7 +78,6 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
args = args,
environment = environment
)
throwGradleExceptionIfError(exitCode)
}
private val isAtLeastJava9: Boolean
@@ -24,6 +24,7 @@ import org.gradle.tooling.provider.model.ToolingModelBuilderRegistry
import org.jetbrains.kotlin.gradle.dsl.KotlinJvmOptionsImpl
import org.jetbrains.kotlin.gradle.dsl.KotlinSingleJavaTargetExtension
import org.jetbrains.kotlin.gradle.dsl.kotlinExtension
import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin
import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKaptGeneratedClassesDir
import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin
import org.jetbrains.kotlin.gradle.internal.KaptVariantData
@@ -22,6 +22,7 @@ import org.gradle.api.internal.HasConvention
import org.gradle.api.logging.Logger
import org.gradle.api.plugins.ExtensionAware
import org.gradle.api.tasks.compile.AbstractCompile
import org.jetbrains.kotlin.compilerRunner.KotlinLogger
import java.io.File
internal fun AbstractCompile.appendClasspathDynamically(file: File) {
@@ -76,4 +77,11 @@ internal inline fun Logger.kotlinDebug(message: () -> String) {
if (isDebugEnabled) {
kotlinDebug(message())
}
}
internal inline fun KotlinLogger.kotlinDebug(fn: () -> String) {
if (isDebugEnabled) {
val msg = fn()
debug("[KOTLIN] $msg")
}
}
@@ -82,13 +82,10 @@ class KotlinMetadataTargetPreset(
override val platformType: KotlinPlatformType
get() = KotlinPlatformType.common
override fun buildCompilationProcessor(compilation: KotlinCommonCompilation): KotlinSourceSetProcessor<*> =
KotlinCommonSourceSetProcessor(
project,
compilation,
KotlinTasksProvider(compilation.target.targetName),
kotlinPluginVersion
)
override fun buildCompilationProcessor(compilation: KotlinCommonCompilation): KotlinSourceSetProcessor<*> {
val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
return KotlinCommonSourceSetProcessor(project, compilation, tasksProvider, kotlinPluginVersion)
}
companion object {
const val PRESET_NAME = "metadata"
@@ -131,8 +128,10 @@ class KotlinJvmTargetPreset(
override val platformType: KotlinPlatformType
get() = KotlinPlatformType.jvm
override fun buildCompilationProcessor(compilation: KotlinJvmCompilation): KotlinSourceSetProcessor<*> =
Kotlin2JvmSourceSetProcessor(project, KotlinTasksProvider(compilation.target.targetName), compilation, kotlinPluginVersion)
override fun buildCompilationProcessor(compilation: KotlinJvmCompilation): KotlinSourceSetProcessor<*> {
val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
return Kotlin2JvmSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
}
companion object {
const val PRESET_NAME = "jvm"
@@ -158,8 +157,10 @@ class KotlinJsTargetPreset(
override val platformType: KotlinPlatformType
get() = KotlinPlatformType.js
override fun buildCompilationProcessor(compilation: KotlinJsCompilation): KotlinSourceSetProcessor<*> =
Kotlin2JsSourceSetProcessor(project, KotlinTasksProvider(compilation.target.targetName), compilation, kotlinPluginVersion)
override fun buildCompilationProcessor(compilation: KotlinJsCompilation): KotlinSourceSetProcessor<*> {
val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
return Kotlin2JsSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
}
companion object {
const val PRESET_NAME = "js"
@@ -179,7 +180,7 @@ class KotlinAndroidTargetPreset(
}
KotlinAndroidPlugin.applyToTarget(
project, result, AndroidTasksProvider(name),
project, result, AndroidTasksProvider(name, useWorkersForCompilation = true),
kotlinPluginVersion
)
@@ -206,7 +207,8 @@ class KotlinJvmWithJavaTargetPreset(
}
AbstractKotlinPlugin.configureTarget(target) { compilation ->
Kotlin2JvmSourceSetProcessor(project, KotlinTasksProvider(name), compilation, kotlinPluginVersion)
val tasksProvider = KotlinTasksProvider(name, useWorkersForCompilation = true)
Kotlin2JvmSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
}
target.compilations.all { compilation ->
@@ -0,0 +1,26 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.plugin
import org.slf4j.Logger
internal inline fun Logger.kotlinDebug(message: () -> String) {
if (isDebugEnabled) {
kotlinDebug(message())
}
}
internal fun Logger.kotlinInfo(message: String) {
this.info("[KOTLIN] $message")
}
internal fun Logger.kotlinDebug(message: String) {
this.debug("[KOTLIN] $message")
}
internal fun Logger.kotlinWarn(message: String) {
this.warn("[KOTLIN] $message")
}
@@ -67,9 +67,8 @@ internal open class KotlinCompileCommon : AbstractKotlinCompile<K2MetadataCompil
override fun callCompiler(args: K2MetadataCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) {
val messageCollector = GradleMessageCollector(logger)
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = GradleCompilerRunner(project)
val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, args)
val exitCode = compilerRunner.runMetadataCompiler(sourceRoots.kotlinSourceFiles, args, environment)
throwGradleExceptionIfError(exitCode)
val compilerRunner = compilerRunner()
val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector)
compilerRunner.runMetadataCompiler(sourceRoots.kotlinSourceFiles, args, environment)
}
}
@@ -15,6 +15,7 @@ import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.compile.AbstractCompile
import org.gradle.api.tasks.compile.JavaCompile
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
@@ -41,6 +42,7 @@ import org.jetbrains.kotlin.incremental.destinationAsFile
import org.jetbrains.kotlin.utils.LibraryUtils
import java.io.File
import java.util.*
import javax.inject.Inject
import kotlin.properties.Delegates
const val KOTLIN_BUILD_DIR_NAME = "kotlin"
@@ -248,6 +250,9 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
}
}
internal open fun compilerRunner(): GradleCompilerRunner =
GradleCompilerRunner(project)
override fun compile() {
assert(false, { "unexpected call to compile()" })
}
@@ -381,31 +386,25 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val messageCollector = GradleMessageCollector(logger)
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = GradleCompilerRunner(project)
val compilerRunner = compilerRunner()
val environment = when {
!incremental ->
GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, args)
else -> {
logger.info(USING_INCREMENTAL_COMPILATION_MESSAGE)
GradleIncrementalCompilerEnvironment(
computedCompilerClasspath,
if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
taskBuildDirectory,
messageCollector, outputItemCollector, args,
usePreciseJavaTracking = usePreciseJavaTracking,
localStateDirs = outputDirectories,
multiModuleICSettings = multiModuleICSettings
)
}
}
if (!incremental) {
val icEnv = if (incremental) {
logger.info(USING_INCREMENTAL_COMPILATION_MESSAGE)
IncrementalCompilationEnvironment(
if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
taskBuildDirectory,
usePreciseJavaTracking = usePreciseJavaTracking,
localStateDirs = outputDirectories,
multiModuleICSettings = multiModuleICSettings
)
} else {
clearOutputDirectories(reason = "IC is disabled for the task")
null
}
try {
val exitCode = compilerRunner.runJvmCompiler(
val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, icEnv)
compilerRunner.runJvmCompiler(
sourceRoots.kotlinSourceFiles,
commonSourceSet.toList(),
sourceRoots.javaSourceRoots,
@@ -415,7 +414,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
)
disableMultiModuleICIfNeeded()
processCompilerExitCode(exitCode)
} catch (e: Throwable) {
cleanupOnError()
throw e
@@ -447,14 +445,6 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
destinationDir.deleteRecursively()
}
private fun processCompilerExitCode(exitCode: ExitCode) {
if (exitCode != ExitCode.OK) {
cleanupOnError()
}
throwGradleExceptionIfError(exitCode)
}
// override setSource to track source directory sets and files (for generated android folders)
override fun setSource(sources: Any?) {
sourceRootsContainer.set(sources)
@@ -468,6 +458,27 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
}
}
@CacheableTask
internal open class KotlinCompileWithWorkers @Inject constructor(
@Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
) : KotlinCompile() {
override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
}
@CacheableTask
internal open class Kotlin2JsCompileWithWorkers @Inject constructor(
@Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
) : Kotlin2JsCompile() {
override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
}
@CacheableTask
internal open class KotlinCompileCommonWithWorkers @Inject constructor(
@Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
) : KotlinCompileCommon() {
override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
}
@CacheableTask
open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>(), KotlinJsCompile {
private val kotlinOptionsImpl = KotlinJsOptionsImpl()
@@ -538,28 +549,19 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>(),
val messageCollector = GradleMessageCollector(logger)
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = GradleCompilerRunner(project)
val compilerRunner = compilerRunner()
val environment = when {
incremental -> {
logger.warn(USING_EXPERIMENTAL_JS_INCREMENTAL_COMPILATION_MESSAGE)
GradleIncrementalCompilerEnvironment(
computedCompilerClasspath,
if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
taskBuildDirectory,
messageCollector,
outputItemCollector,
args,
multiModuleICSettings = multiModuleICSettings
)
}
else -> {
GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, args)
}
}
val icEnv = if (incremental) {
logger.warn(USING_EXPERIMENTAL_JS_INCREMENTAL_COMPILATION_MESSAGE)
IncrementalCompilationEnvironment(
if (hasFilesInTaskBuildDirectory()) changedFiles else ChangedFiles.Unknown(),
taskBuildDirectory,
multiModuleICSettings = multiModuleICSettings
)
} else null
val exitCode = compilerRunner.runJsCompiler(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment)
throwGradleExceptionIfError(exitCode)
val environment = GradleCompilerEnvironment(computedCompilerClasspath, messageCollector, outputItemCollector, icEnv)
compilerRunner.runJsCompiler(sourceRoots.kotlinSourceFiles, commonSourceSet.toList(), args, environment)
}
}
@@ -577,7 +579,9 @@ private fun Task.getGradleVersion(): ParsedGradleVersion? {
return result
}
internal class GradleMessageCollector(val logger: Logger) : MessageCollector {
internal class GradleMessageCollector(val logger: KotlinLogger) : MessageCollector {
constructor(logger: Logger) : this(GradleKotlinLogger(logger))
private var hasErrors = false
override fun hasErrors() = hasErrors
@@ -22,25 +22,35 @@ import org.jetbrains.kotlin.gradle.plugin.*
import org.jetbrains.kotlin.gradle.plugin.mpp.defaultSourceSetName
import org.jetbrains.kotlin.gradle.plugin.sources.applyLanguageSettingsToKotlinTask
internal open class KotlinTasksProvider(val targetName: String) {
internal open class KotlinTasksProvider(
val targetName: String,
private val useWorkersForCompilation: Boolean = false
) {
open fun createKotlinJVMTask(
project: Project,
name: String,
compilation: KotlinCompilation
): KotlinCompile =
project.tasks.create(name, KotlinCompile::class.java).apply {
): KotlinCompile {
val taskClass = if (useWorkersForCompilation) KotlinCompileWithWorkers::class.java else KotlinCompile::class.java
return project.tasks.create(name, taskClass).apply {
configure(this, project, compilation)
}
}
fun createKotlinJSTask(project: Project, name: String, compilation: KotlinCompilation): Kotlin2JsCompile =
project.tasks.create(name, Kotlin2JsCompile::class.java).apply {
fun createKotlinJSTask(project: Project, name: String, compilation: KotlinCompilation): Kotlin2JsCompile {
val taskClass = if (useWorkersForCompilation) Kotlin2JsCompileWithWorkers::class.java else Kotlin2JsCompile::class.java
return project.tasks.create(name, taskClass).apply {
configure(this, project, compilation)
}
}
fun createKotlinCommonTask(project: Project, name: String, compilation: KotlinCompilation): KotlinCompileCommon =
project.tasks.create(name, KotlinCompileCommon::class.java).apply {
fun createKotlinCommonTask(project: Project, name: String, compilation: KotlinCompilation): KotlinCompileCommon {
val taskClass = if (useWorkersForCompilation) KotlinCompileCommonWithWorkers::class.java else KotlinCompileCommon::class.java
return project.tasks.create(name, taskClass).apply {
configure(this, project, compilation)
}
}
open fun configure(
kotlinTask: AbstractKotlinCompile<*>,
@@ -64,7 +74,10 @@ internal open class KotlinTasksProvider(val targetName: String) {
RegexTaskToFriendTaskMapper.Default(targetName)
}
internal class AndroidTasksProvider(targetName: String) : KotlinTasksProvider(targetName) {
internal class AndroidTasksProvider(
targetName: String,
useWorkersForCompilation: Boolean = false
) : KotlinTasksProvider(targetName, useWorkersForCompilation = useWorkersForCompilation) {
override val taskToFriendTaskMapper: TaskToFriendTaskMapper =
RegexTaskToFriendTaskMapper.Android(targetName)