Refactor gradle daemon client

This commit is contained in:
Alexey Tsvetkov
2016-12-30 18:26:11 +03:00
parent d4d1d5ad0f
commit c9a874cba9
9 changed files with 251 additions and 151 deletions
@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.daemon.incremental.IncrementalCompilationSeverity.CO
import org.jetbrains.kotlin.daemon.incremental.IncrementalCompilationSeverity.LOGGING
import org.jetbrains.kotlin.incremental.ICReporter
import java.io.File
import java.io.Serializable
internal class RemoteICReporter(
servicesFacade: CompilerServicesFacadeBase,
@@ -39,7 +40,7 @@ internal class RemoteICReporter(
override fun reportCompileIteration(sourceFiles: Collection<File>, exitCode: ExitCode) {
if (shouldReport(COMPILED_FILES.value)) {
report(COMPILED_FILES.value, message = null, attachment = ArrayList(sourceFiles))
report(COMPILED_FILES.value, message = null, attachment = CompileIterationResult(sourceFiles, exitCode.toString()))
}
}
}
@@ -50,4 +51,10 @@ internal class RemoteICReporter(
enum class IncrementalCompilationSeverity(val value: Int) {
COMPILED_FILES(0),
LOGGING(10)
}
class CompileIterationResult(val sourceFiles: Iterable<File>, val exitCode: String) : Serializable {
companion object {
const val serialVersionUID: Long = 0
}
}
@@ -1,8 +1,10 @@
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
import org.jetbrains.kotlin.gradle.tasks.findToolsJar
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.GradleICReporter
@@ -12,8 +14,9 @@ import java.net.URL
internal open class GradleCompilerEnvironment(
val compilerJar: File,
messageCollector: MessageCollector,
outputItemsCollector: OutputItemsCollector
messageCollector: GradleMessageCollector,
outputItemsCollector: OutputItemsCollector,
val compilerArgs: CommonCompilerArguments
) : CompilerEnvironment(Services.EMPTY, messageCollector, outputItemsCollector) {
val toolsJar: File? by lazy { findToolsJar() }
@@ -29,9 +32,10 @@ internal class GradleIncrementalCompilerEnvironment(
val changedFiles: ChangedFiles,
val reporter: GradleICReporter,
val workingDir: File,
messageCollector: MessageCollector,
messageCollector: GradleMessageCollector,
outputItemsCollector: OutputItemsCollector,
val kaptAnnotationsFileUpdater: AnnotationFileUpdater?,
val artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider?,
val artifactFile: File?
) : GradleCompilerEnvironment(compilerJar, messageCollector, outputItemsCollector)
val artifactFile: File?,
compilerArgs: CommonCompilerArguments
) : GradleCompilerEnvironment(compilerJar, messageCollector, outputItemsCollector, compilerArgs)
@@ -0,0 +1,140 @@
package org.jetbrains.kotlin.compilerRunner
import org.gradle.api.Project
import org.gradle.api.logging.Logger
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.incremental.CompileIterationResult
import org.jetbrains.kotlin.daemon.incremental.IncrementalCompilationSeverity
import org.jetbrains.kotlin.daemon.incremental.toDirtyData
import org.jetbrains.kotlin.daemon.incremental.toSimpleDirtyData
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.plugin.kotlinWarn
import org.jetbrains.kotlin.incremental.DirtyData
import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifference
import org.jetbrains.kotlin.incremental.pathsAsStringRelativeTo
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.File
import java.io.Serializable
import java.rmi.Remote
import java.rmi.server.UnicastRemoteObject
internal open class GradleCompilerServicesFacadeImpl(
project: Project,
val compilerMessageCollector: MessageCollector,
port: Int = SOCKET_ANY_FREE_PORT
) : UnicastRemoteObject(port), CompilerServicesFacadeBase, Remote {
protected val log: Logger = project.logger
override fun report(category: ReportCategory, severity: Int, message: String?, attachment: Serializable?) {
when (category) {
ReportCategory.COMPILER_MESSAGE -> {
val compilerSeverity = CompilerMessageSeverity.values().firstOrNull { it.value == severity }
if (compilerSeverity != null && message != null && attachment is CompilerMessageLocation) {
compilerMessageCollector.report(compilerSeverity, message, attachment)
}
else {
reportUnexpectedMessage(category, severity, message, attachment)
}
}
ReportCategory.DAEMON_MESSAGE -> {
log.kotlinDebug { "[DAEMON] $message" }
}
else -> {
reportUnexpectedMessage(category, severity, message, attachment)
}
}
}
protected fun reportUnexpectedMessage(category: ReportCategory, severity: Int, message: String?, attachment: Serializable?) {
// todo add assert to tests
log.kotlinWarn("Received unexpected message from compiler daemon: category=$category, severity=$severity, message='$message', attachment=$attachment")
}
}
internal class GradleIncrementalCompilerServicesFacadeImpl(
project: Project,
private val environment: GradleIncrementalCompilerEnvironment,
port: Int = SOCKET_ANY_FREE_PORT
) : GradleCompilerServicesFacadeImpl(project, environment.messageCollector, port),
IncrementalCompilerServicesFacade {
private val projectRootFile = project.rootProject.projectDir
override fun report(category: ReportCategory, severity: Int, message: String?, attachment: Serializable?) {
when (category) {
ReportCategory.INCREMENTAL_COMPILATION -> {
val icSeverity = IncrementalCompilationSeverity.values().firstOrNull { it.value == severity }
when (icSeverity) {
IncrementalCompilationSeverity.COMPILED_FILES -> {
@Suppress("UNCHECKED_CAST")
val compileIterationResult = attachment as? CompileIterationResult
if (compileIterationResult == null) {
reportUnexpectedMessage(category, severity, message, attachment)
}
else {
val sourceFiles = compileIterationResult.sourceFiles
if (sourceFiles.any()) {
log.kotlinDebug { "compile iteration: ${sourceFiles.pathsAsStringRelativeTo(projectRootFile)}" }
}
val exitCode = compileIterationResult.exitCode
log.kotlinDebug { "compiler exit code: $exitCode" }
}
}
IncrementalCompilationSeverity.LOGGING -> {
log.kotlinDebug { "[IC] $message" }
}
else -> {
reportUnexpectedMessage(category, severity, message, attachment)
}
}
}
else -> {
super.report(category, severity, message, attachment)
}
}
}
override fun hasAnnotationsFileUpdater(): Boolean =
environment.kaptAnnotationsFileUpdater != null
override fun updateAnnotations(outdatedClassesJvmNames: Iterable<String>) {
val jvmNames = outdatedClassesJvmNames.map { JvmClassName.byInternalName(it) }
environment.kaptAnnotationsFileUpdater!!.updateAnnotations(jvmNames)
}
override fun revert() {
environment.kaptAnnotationsFileUpdater!!.revert()
}
override fun getChanges(artifact: File, sinceTS: Long): Iterable<SimpleDirtyData>? {
val artifactChanges = environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry[artifact]
} ?: return null
val (beforeLastBuild, afterLastBuild) = artifactChanges.partition { it.buildTS < sinceTS }
if (beforeLastBuild.isEmpty()) return null
return afterLastBuild.map { it.dirtyData.toSimpleDirtyData() }
}
override fun registerChanges(timestamp: Long, dirtyData: SimpleDirtyData) {
val artifactFile = environment.artifactFile ?: return
environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry.add(artifactFile, ArtifactDifference(timestamp, dirtyData.toDirtyData()))
}
}
override fun unknownChanges(timestamp: Long) {
val artifactFile = environment.artifactFile ?: return
environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry.remove(artifactFile)
registry.add(artifactFile, ArtifactDifference(timestamp, DirtyData()))
}
}
}
@@ -27,14 +27,11 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.client.RemoteOutputStreamServer
import org.jetbrains.kotlin.daemon.common.CompilerId
import org.jetbrains.kotlin.daemon.common.SOCKET_ANY_FREE_PORT
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.incremental.IncrementalCompilationSeverity
import org.jetbrains.kotlin.gradle.plugin.ParentLastURLClassLoader
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import org.jetbrains.kotlin.gradle.tasks.SourceRoots
import org.jetbrains.kotlin.incremental.classpathAsList
import org.jetbrains.kotlin.incremental.destinationAsFile
import org.jetbrains.kotlin.incremental.makeModuleFile
import org.jetbrains.kotlin.incremental.*
import java.io.*
import kotlin.concurrent.thread
@@ -137,10 +134,10 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
override fun compileWithDaemon(compilerClassName: String, argsArray: Array<String>, environment: GradleCompilerEnvironment): ExitCode? {
val exitCode = if (environment is GradleIncrementalCompilerEnvironment) {
performIncrementalCompilationWithDaemon(argsArray, environment)
incrementalCompilationWithDaemon(environment)
}
else {
super.compileWithDaemon(compilerClassName, argsArray, environment)
nonIncrementalCompilationWithDaemon(compilerClassName, environment)
}
exitCode?.let {
withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
@@ -151,16 +148,58 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
return exitCode
}
private fun performIncrementalCompilationWithDaemon(argsArray: Array<String>, environment: GradleIncrementalCompilerEnvironment): ExitCode? {
val port = SOCKET_ANY_FREE_PORT
val services = IncrementalCompilationFacadeImpl(environment, port)
val compilerOut = ByteArrayOutputStream()
val daemonOut = ByteArrayOutputStream()
private fun nonIncrementalCompilationWithDaemon(
compilerClassName: String,
environment: GradleCompilerEnvironment
): ExitCode? {
val targetPlatform = when (compilerClassName) {
K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
K2JS_COMPILER -> CompileService.TargetPlatform.JS
K2METADATA_COMPILER -> CompileService.TargetPlatform.METADATA
else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
}
val res = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
val remoteCompilerOut = RemoteOutputStreamServer(compilerOut, port)
val remoteDaemonOut = RemoteOutputStreamServer(daemonOut, port)
daemon.serverSideJvmIC(sessionId, argsArray, services, remoteCompilerOut, remoteDaemonOut, null)
daemon.compile(
sessionId,
CompileService.CompilerMode.NON_INCREMENTAL_COMPILER,
targetPlatform,
environment.compilerArgs,
AdditionalCompilerArguments(reportingFilters = getNonIncrementalReportingFilters(environment.compilerArgs.verbose)),
GradleCompilerServicesFacadeImpl(project, environment.messageCollector),
operationsTracer = null)
}
val exitCode = res?.get()?.let { exitCodeFromProcessExitCode(it) }
if (exitCode == null) {
log.warn("Could not perform non-incremental compilation with Kotlin compile daemon, " +
"non-incremental compilation in fallback mode will be performed")
}
return exitCode
}
private fun incrementalCompilationWithDaemon(environment: GradleIncrementalCompilerEnvironment): ExitCode? {
val knownChangedFiles = environment.changedFiles as? ChangedFiles.Known
val additionalCompilerArguments = IncrementalCompilerArguments(
areFileChangesKnown = knownChangedFiles != null,
modifiedFiles = knownChangedFiles?.modified,
deletedFiles = knownChangedFiles?.removed,
workingDir = environment.workingDir,
customCacheVersion = GRADLE_CACHE_VERSION,
customCacheVersionFileName = GRADLE_CACHE_VERSION_FILE_NAME,
reportingFilters = getNonIncrementalReportingFilters(environment.compilerArgs.verbose) +
getIncrementalReportingFilters(environment.compilerArgs.verbose)
)
val res = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
daemon.compile(
sessionId,
CompileService.CompilerMode.INCREMENTAL_COMPILER,
CompileService.TargetPlatform.JVM,
environment.compilerArgs,
additionalCompilerArguments,
GradleIncrementalCompilerServicesFacadeImpl(project, environment),
operationsTracer = null)
}
val exitCode = res?.get()?.let { exitCodeFromProcessExitCode(it) }
@@ -168,13 +207,40 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
log.warn("Could not perform incremental compilation with Kotlin compile daemon, " +
"non-incremental compilation in fallback mode will be performed")
}
processCompilerOutput(environment, compilerOut, exitCode)
BufferedReader(StringReader(daemonOut.toString())).forEachLine {
environment.messageCollector.report(CompilerMessageSeverity.INFO, it, CompilerMessageLocation.NO_LOCATION)
}
return exitCode
}
private fun getNonIncrementalReportingFilters(verbose: Boolean): List<ReportingFilter> {
val result = ArrayList<ReportingFilter>()
val compilerMessagesSeverities = ArrayList<Int>().apply {
add(CompilerMessageSeverity.ERROR.value)
add(CompilerMessageSeverity.EXCEPTION.value)
add(CompilerMessageSeverity.WARNING.value)
add(CompilerMessageSeverity.INFO.value)
if (verbose) {
add(CompilerMessageSeverity.OUTPUT.value)
add(CompilerMessageSeverity.LOGGING.value)
}
}
if (verbose) {
result.add(ReportingFilter(ReportCategory.DAEMON_MESSAGE, emptyList()))
}
val compilerMessagesFilter = ReportingFilter(ReportCategory.COMPILER_MESSAGE, compilerMessagesSeverities)
result.add(compilerMessagesFilter)
return result
}
private fun getIncrementalReportingFilters(verbose: Boolean) =
if (verbose) {
listOf(ReportingFilter(ReportCategory.INCREMENTAL_COMPILATION, listOf(IncrementalCompilationSeverity.COMPILED_FILES.value, IncrementalCompilationSeverity.LOGGING.value)))
}
else emptyList()
private fun compileOutOfProcess(
argsArray: Array<String>,
compilerClassName: String,
@@ -1,115 +0,0 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.compilerRunner
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.daemon.common.IncrementalCompilationServicesFacade
import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface
import org.jetbrains.kotlin.daemon.common.SOCKET_ANY_FREE_PORT
import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
import org.jetbrains.kotlin.daemon.incremental.toDirtyData
import org.jetbrains.kotlin.daemon.incremental.toSimpleDirtyData
import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.DirtyData
import org.jetbrains.kotlin.incremental.GRADLE_CACHE_VERSION
import org.jetbrains.kotlin.incremental.GRADLE_CACHE_VERSION_FILE_NAME
import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifference
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.File
import java.rmi.server.UnicastRemoteObject
internal class IncrementalCompilationFacadeImpl(
private val environment: GradleIncrementalCompilerEnvironment,
port: Int = SOCKET_ANY_FREE_PORT
) : IncrementalCompilationServicesFacade,
UnicastRemoteObject(port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) {
override fun areFileChangesKnown(): Boolean {
return environment.changedFiles is ChangedFiles.Known
}
override fun deletedFiles(): List<File> {
return (environment.changedFiles as ChangedFiles.Known).removed
}
override fun modifiedFiles(): List<File> {
return (environment.changedFiles as ChangedFiles.Known).modified
}
override fun reportCompileIteration(files: Iterable<File>, exitCode: Int) {
environment.reporter.reportCompileIteration(files, ExitCode.values().first { it.code == exitCode })
}
override fun reportIC(message: String) {
environment.reporter.report { message }
}
override fun shouldReportIC(): Boolean {
return environment.reporter.isDebugEnabled
}
override fun workingDir(): File {
return environment.workingDir
}
override fun customCacheVersion(): Int =
GRADLE_CACHE_VERSION
override fun customCacheVersionFileName(): String =
GRADLE_CACHE_VERSION_FILE_NAME
override fun hasAnnotationsFileUpdater(): Boolean {
return environment.kaptAnnotationsFileUpdater != null
}
override fun revert() {
environment.kaptAnnotationsFileUpdater!!.revert()
}
override fun updateAnnotations(outdatedClassesJvmNames: Iterable<String>) {
val jvmNames = outdatedClassesJvmNames.map { JvmClassName.byInternalName(it) }
environment.kaptAnnotationsFileUpdater!!.updateAnnotations(jvmNames)
}
override fun getChanges(artifact: File, sinceTS: Long): Iterable<SimpleDirtyData>? {
val artifactChanges = environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry[artifact]
} ?: return null
val (beforeLastBuild, afterLastBuild) = artifactChanges.partition { it.buildTS < sinceTS }
if (beforeLastBuild.isEmpty()) return null
return afterLastBuild.map { it.dirtyData.toSimpleDirtyData() }
}
override fun registerChanges(timestamp: Long, dirtyData: SimpleDirtyData) {
val artifactFile = environment.artifactFile ?: return
environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry.add(artifactFile, ArtifactDifference(timestamp, dirtyData.toDirtyData()))
}
}
override fun unknownChanges(timestamp: Long) {
val artifactFile = environment.artifactFile ?: return
environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
registry.remove(artifactFile)
registry.add(artifactFile, ArtifactDifference(timestamp, DirtyData()))
}
}
}
@@ -62,7 +62,7 @@ open class KaptTask : AbstractCompile() {
val messageCollector = GradleMessageCollector(logger)
val outputItemCollector = OutputItemsCollectorImpl()
val environment = GradleCompilerEnvironment(kotlinCompileTask.compilerJar, messageCollector, outputItemCollector)
val environment = GradleCompilerEnvironment(kotlinCompileTask.compilerJar, messageCollector, outputItemCollector, args)
if (environment.toolsJar == null) {
throw GradleException("Could not find tools.jar in system classpath, which is required for kapt to work")
}
@@ -49,7 +49,7 @@ internal open class KotlinCompileCommon : AbstractKotlinCompile<K2MetadataCompil
val messageCollector = GradleMessageCollector(logger)
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = GradleCompilerRunner(project)
val environment = GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector)
val environment = GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector, args)
val exitCode = compilerRunner.runMetadataCompiler(sourceRoots.kotlinSourceFiles, args, environment)
throwGradleExceptionIfError(exitCode)
}
@@ -44,6 +44,7 @@ import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistry
import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider
import org.jetbrains.kotlin.utils.LibraryUtils
import java.io.File
import java.rmi.Remote
import java.util.*
import kotlin.properties.Delegates
@@ -200,13 +201,13 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
val reporter = GradleICReporter(project.rootProject.projectDir)
val environment = when {
!incremental -> GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector)
!incremental -> GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector, args)
else -> {
logger.warn(USING_EXPERIMENTAL_INCREMENTAL_MESSAGE)
GradleIncrementalCompilerEnvironment(compilerJar, changedFiles, reporter, taskBuildDirectory,
messageCollector, outputItemCollector, kaptAnnotationsFileUpdater,
artifactDifferenceRegistryProvider,
artifactFile)
artifactFile, args)
}
}
@@ -331,7 +332,7 @@ open class Kotlin2JsCompile() : AbstractKotlinCompile<K2JSCompilerArguments>(),
val outputItemCollector = OutputItemsCollectorImpl()
val compilerRunner = GradleCompilerRunner(project)
val environment = GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector)
val environment = GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector, args)
val exitCode = compilerRunner.runJsCompiler(sourceRoots.kotlinSourceFiles, args, environment)
throwGradleExceptionIfError(exitCode)
}
@@ -21,12 +21,9 @@ import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
import java.io.File
internal class GradleICReporter(private val projectRootFile: File) : ICReporter() {
internal class GradleICReporter(private val projectRootFile: File) : ICReporter {
private val log = Logging.getLogger(GradleICReporter::class.java)
val isDebugEnabled: Boolean
get() = log.isDebugEnabled
override fun report(message: ()->String) {
log.kotlinDebug(message)
}
@@ -34,7 +31,7 @@ internal class GradleICReporter(private val projectRootFile: File) : ICReporter(
override fun pathsAsString(files: Iterable<File>): String =
files.pathsAsStringRelativeTo(projectRootFile)
override fun reportCompileIteration(sourceFiles: Iterable<File>, exitCode: ExitCode) {
override fun reportCompileIteration(sourceFiles: Collection<File>, exitCode: ExitCode) {
if (sourceFiles.any()) {
report { "compile iteration: ${pathsAsString(sourceFiles)}" }
}