jps: support multiplatform incremental compilation for jvm and js

- support common modules metadata compilation under flag (it is not required since all common source roots are included transitively for now)
- introduce expect actual tracker in jps: move implementation from gradle to build-common
- support js incremental compilation: move implementation from gradle to build-common

Original commit: 0eee2729cd
This commit is contained in:
Sergey Rostov
2018-04-03 13:00:06 +03:00
parent a64a2f694f
commit 5f818789cf
72 changed files with 5649 additions and 731 deletions
@@ -88,10 +88,18 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
k2MetadataArguments: K2MetadataCompilerArguments,
compilerSettings: CompilerSettings,
environment: JpsCompilerEnvironment,
destination: String,
classpath: Collection<String>,
sourceFiles: Collection<File>
) {
val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2MetadataArguments))
val classpathSet = arguments.classpath?.split(File.pathSeparator)?.toMutableSet() ?: mutableSetOf()
classpathSet.addAll(classpath)
arguments.classpath = classpath.joinToString(File.pathSeparator)
arguments.freeArgs = sourceFiles.map { it.absolutePath }
arguments.destination = arguments.destination ?: destination
withCompilerSettings(compilerSettings) {
runCompiler(K2METADATA_COMPILER, arguments, environment)
}
@@ -19,9 +19,7 @@ package org.jetbrains.kotlin.jps.build
import com.intellij.openapi.diagnostic.Logger
import com.intellij.util.containers.ContainerUtil
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.BuildRootIndex
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.BuildTargetIndex
import org.jetbrains.jps.builders.java.dependencyView.Mappings
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.FSOperations
@@ -43,7 +41,7 @@ class FSOperationsHelper(
fun markChunk(recursively: Boolean, kotlinOnly: Boolean, excludeFiles: Set<File> = setOf()) {
fun shouldMark(file: File): Boolean {
if (kotlinOnly && !KotlinSourceFileCollector.isKotlinSourceFile(file)) return false
if (kotlinOnly && !file.isKotlinSourceFile) return false
if (file in excludeFiles) return false
@@ -17,11 +17,7 @@
package org.jetbrains.kotlin.jps.build
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.util.Key
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.containers.ContainerUtil
import com.intellij.util.containers.MultiMap
import gnu.trove.THashSet
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.DirtyFilesHolder
@@ -32,44 +28,35 @@ import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.incremental.*
import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.*
import org.jetbrains.jps.incremental.java.JavaBuilder
import org.jetbrains.jps.incremental.messages.BuildMessage
import org.jetbrains.jps.incremental.messages.CompilerMessage
import org.jetbrains.jps.incremental.messages.ProgressMessage
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.model.JpsProject
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.JvmBuildMetaInfo
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.INFO
import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil
import org.jetbrains.kotlin.compilerRunner.*
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.config.CompilerRunnerConstants.INTERNAL_ERROR_PREFIX
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.isDaemonEnabled
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments
import org.jetbrains.kotlin.jps.incremental.*
import org.jetbrains.kotlin.jps.platforms.KotlinCommonModuleBuildTarget
import org.jetbrains.kotlin.jps.platforms.KotlinJsModuleBuildTarget
import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget
import org.jetbrains.kotlin.jps.platforms.kotlinBuildTargets
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.preloading.ClassCondition
import org.jetbrains.kotlin.progress.CompilationCanceledException
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
import org.jetbrains.kotlin.utils.KotlinPaths
import org.jetbrains.kotlin.utils.KotlinPathsFromHomeDir
import org.jetbrains.kotlin.utils.PathUtil
import org.jetbrains.kotlin.utils.keysToMap
import org.jetbrains.org.objectweb.asm.ClassReader
import org.jetbrains.kotlin.utils.keysToMapExceptNulls
import java.io.File
import java.util.*
import kotlin.collections.HashSet
@@ -167,29 +154,39 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
context: CompileContext,
fsOperations: FSOperationsHelper
) {
val dirtyFilesHolder = object : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
FSOperations.processFilesToRecompile(context, chunk, processor)
val roundDirtyFiles = KotlinRoundDirtySourceFilesHolder(
chunk,
context,
fsOperations,
object : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
FSOperations.processFilesToRecompile(context, chunk, processor)
}
}
}
val chunkDirtyFiles = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder)
val chunkRemovedFiles = chunk.targets.keysToMap { KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, it) }
)
val representativeTarget = context.kotlinBuildTargets[chunk.representativeTarget()] ?: return
val incrementalCaches = getIncrementalCaches(chunk, context)
val messageCollector = MessageCollectorAdapter(context)
val environment = createCompileEnvironment(incrementalCaches, LookupTracker.DO_NOTHING, context, chunk, messageCollector)
if (environment == null) return
val messageCollector = MessageCollectorAdapter(context, representativeTarget)
val environment = createCompileEnvironment(
representativeTarget,
incrementalCaches,
LookupTracker.DO_NOTHING,
ExpectActualTracker.DoNothing,
chunk,
messageCollector
) ?: return
val removedClasses = HashSet<String>()
for (target in chunk.targets) {
val cache = incrementalCaches[target]!!
val dirtyFiles = chunkDirtyFiles[target]
val removedFiles = chunkRemovedFiles[target] ?: emptyList()
val cache = incrementalCaches[target] ?: continue
val dirtyFiles = roundDirtyFiles.getDirtyFiles(target)
val removedFiles = roundDirtyFiles.getRemovedFilesSet(target)
val existingClasses = JpsKotlinCompilerRunner().classesFqNamesByFiles(environment, dirtyFiles.toHashSet())
val previousClasses = cache.classesBySources(dirtyFiles + removedFiles)
val existingClasses = JpsKotlinCompilerRunner().classesFqNamesByFiles(environment, dirtyFiles)
val previousClasses = cache.classesFqNamesBySources(dirtyFiles + removedFiles)
for (jvmClassName in previousClasses) {
val fqName = jvmClassName.fqNameForClassNameWithoutDollars.asString()
val fqName = jvmClassName.asString()
if (fqName !in existingClasses) {
removedClasses.add(fqName)
}
@@ -214,50 +211,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val allVersions = cacheVersionsProvider.allVersions(targets)
val actions = allVersions.map { it.checkVersion() }.toMutableSet()
val kotlinModuleBuilderTarget = context.kotlinBuildTargets[chunk.representativeTarget()]
if (kotlinModuleBuilderTarget !is KotlinJsModuleBuildTarget) {
val args = compilerArgumentsForChunk(chunk)
val currentBuildMetaInfo = JvmBuildMetaInfo(args)
for (target in chunk.targets) {
val file = jvmBuildMetaInfoFile(target, dataManager)
if (!file.exists()) continue
val lastBuildMetaInfo =
try {
JvmBuildMetaInfo.deserializeFromString(file.readText()) ?: continue
} catch (e: Exception) {
LOG.error("Could not deserialize jvm build meta info", e)
continue
}
val lastBuildLangVersion = LanguageVersion.fromVersionString(lastBuildMetaInfo.languageVersionString)
val lastBuildApiVersion = ApiVersion.parse(lastBuildMetaInfo.apiVersionString)
val currentLangVersion =
args.languageVersion?.let { LanguageVersion.fromVersionString(it) } ?: LanguageVersion.LATEST_STABLE
val currentApiVersion =
args.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.createByLanguageVersion(currentLangVersion)
val reasonToRebuild = when {
currentLangVersion != lastBuildLangVersion -> {
"Language version was changed ($lastBuildLangVersion -> $currentLangVersion)"
}
currentApiVersion != lastBuildApiVersion -> {
"Api version was changed ($lastBuildApiVersion -> $currentApiVersion)"
}
lastBuildLangVersion != LanguageVersion.KOTLIN_1_0 && lastBuildMetaInfo.isEAP && !currentBuildMetaInfo.isEAP -> {
// If EAP->Non-EAP build with IC, then rebuild all kotlin
"Last build was compiled with EAP-plugin"
}
else -> null
}
if (reasonToRebuild != null) {
LOG.info("$reasonToRebuild. Performing non-incremental rebuild (kotlin only)")
actions.add(CacheVersion.Action.REBUILD_ALL_KOTLIN)
}
}
}
val kotlinModuleBuilderTarget = context.kotlinBuildTargets[chunk.representativeTarget()]!!
kotlinModuleBuilderTarget.checkCachesVersions(chunk, dataManager, actions)
return actions
}
@@ -276,14 +231,20 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
outputConsumer: ModuleLevelBuilder.OutputConsumer
): ModuleLevelBuilder.ExitCode {
JavaBuilderUtil.registerFilterToSkipMarkingAffectedFileDirty(context) {
it.isKotlinSourceFile
}
if (chunk.isDummy(context))
return NOTHING_DONE
val messageCollector = MessageCollectorAdapter(context)
val kotlinTarget = context.kotlinBuildTargets[chunk.representativeTarget()] ?: return OK
val messageCollector = MessageCollectorAdapter(context, kotlinTarget)
val fsOperations = FSOperationsHelper(context, chunk, LOG)
try {
val proposedExitCode = doBuild(chunk, context, dirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
val proposedExitCode = doBuild(chunk, kotlinTarget, context, dirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
val actualExitCode = if (proposedExitCode == OK && fsOperations.hasMarkedDirty) ADDITIONAL_PASS_REQUIRED else proposedExitCode
@@ -304,6 +265,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
private fun doBuild(
chunk: ModuleChunk,
representativeTarget: KotlinModuleBuildTarget,
context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
messageCollector: MessageCollectorAdapter,
@@ -311,12 +273,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
fsOperations: FSOperationsHelper
): ModuleLevelBuilder.ExitCode {
// Workaround for Android Studio
val isJsModule = context.kotlinBuildTargets[chunk.representativeTarget()] is KotlinJsModuleBuildTarget
val isJsModule = representativeTarget is KotlinJsModuleBuildTarget
if (!JavaBuilder.IS_ENABLED[context, true] && !isJsModule) {
messageCollector.report(INFO, "Kotlin JPS plugin is disabled")
return NOTHING_DONE
}
val roundDirtyFilesHolder = KotlinRoundDirtySourceFilesHolder(chunk, context, fsOperations, dirtyFilesHolder)
val projectDescriptor = context.projectDescriptor
val dataManager = projectDescriptor.dataManager
val targets = chunk.targets
@@ -325,7 +288,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)
|| targets.any { rebuildAfterCacheVersionChanged[it] == true }
if (hasKotlinDirtyOrRemovedFiles(dirtyFilesHolder, chunk)) {
if (roundDirtyFilesHolder.hasDirtyOrRemovedFiles) {
if (!isChunkRebuilding && !IncrementalCompilation.isEnabled()) {
targets.forEach { rebuildAfterCacheVersionChanged[it] = true }
return CHUNK_REBUILD_REQUIRED
@@ -347,23 +310,30 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val project = projectDescriptor.project
val lookupTracker = getLookupTracker(project)
val exceptActualTracer = ExpectActualTrackerImpl()
val incrementalCaches = getIncrementalCaches(chunk, context)
val environment = createCompileEnvironment(incrementalCaches, lookupTracker, context, chunk, messageCollector) ?: return ABORT
val environment = createCompileEnvironment(
representativeTarget,
incrementalCaches,
lookupTracker,
exceptActualTracer,
chunk,
messageCollector
) ?: return ABORT
val commonArguments = compilerArgumentsForChunk(chunk).apply {
val commonArguments = representativeTarget.compilerArgumentsForChunk(chunk).apply {
reportOutputFiles = true
version = true // Always report the version to help diagnosing user issues if they submit the compiler output
}
val allCompiledFiles = getAllCompiledFilesContainer(context)
val filesToCompile = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder)
LOG.debug("Compiling files: ${filesToCompile.values()}")
if (LOG.isDebugEnabled) {
LOG.debug("Compiling files: ${roundDirtyFilesHolder.dirtyFiles}")
}
val start = System.nanoTime()
val outputItemCollector = doCompileModuleChunk(
allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder,
environment, filesToCompile, incrementalCaches, fsOperations
chunk, representativeTarget, commonArguments, context, roundDirtyFilesHolder, environment,
incrementalCaches
)
statisticsLogger.registerStatistic(chunk, System.nanoTime() - start)
@@ -386,7 +356,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
saveVersions(context, chunk, commonArguments)
if (targets.any { hasKotlin[it] == null }) {
fsOperations.markChunk(recursively = false, kotlinOnly = true, excludeFiles = filesToCompile.values().toSet())
fsOperations.markChunk(recursively = false, kotlinOnly = true, excludeFiles = roundDirtyFilesHolder.dirtyFiles)
}
for (target in targets) {
@@ -398,7 +368,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
context.kotlinBuildTargets[it]?.doAfterBuild()
}
updateJavaMappings(chunk, context, dirtyFilesHolder, filesToCompile, generatedFiles, incrementalCaches)
representativeTarget.updateChunkMappings(
chunk,
roundDirtyFilesHolder,
generatedFiles,
incrementalCaches
)
if (!IncrementalCompilation.isEnabled()) {
return OK
@@ -410,14 +385,17 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
progress.progress("updating IC caches")
val changesCollector = ChangesCollector()
for ((target, files) in generatedFiles) {
updateIncrementalCache(files, incrementalCaches[target]!!, changesCollector, null)
val kotlinModuleBuilderTarget = context.kotlinBuildTargets[target]!!
kotlinModuleBuilderTarget.updateCaches(incrementalCaches[target]!!, files, changesCollector, environment)
}
updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile)
updateLookupStorage(lookupTracker, dataManager, roundDirtyFilesHolder)
if (!isChunkRebuilding) {
changesCollector.processChangesUsingLookups(
filesToCompile.values().toSet(),
roundDirtyFilesHolder.dirtyFiles,
dataManager,
fsOperations,
incrementalCaches.values
@@ -451,8 +429,9 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
LOG.info("Clearing caches for " + targets.joinToString { it.presentableName })
val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager)
val kotlinBuildTargets = context.kotlinBuildTargets
for (target in targets) {
dataManager.getKotlinCache(target).clean()
dataManager.getKotlinCache(kotlinBuildTargets[target]!!).clean()
hasKotlin.clean(target)
rebuildAfterCacheVersionChanged[target] = true
}
@@ -464,8 +443,9 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
CacheVersion.Action.CLEAN_NORMAL_CACHES -> {
LOG.info("Clearing caches for all targets")
val kotlinBuildTargets = context.kotlinBuildTargets
for (target in context.allTargets()) {
dataManager.getKotlinCache(target).clean()
dataManager.getKotlinCache(kotlinBuildTargets[target]!!).clean()
}
}
CacheVersion.Action.CLEAN_DATA_CONTAINER -> {
@@ -491,12 +471,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager)
for (sourceRoot in sourceRoots) {
val ktFiles = sourceRoot.file.walk().filter { KotlinSourceFileCollector.isKotlinSourceFile(it) }
val ktFiles = sourceRoot.file.walk().filter { it.isKotlinSourceFile }
fsOperations.markFiles(ktFiles.toList())
}
val kotlinBuildTargets = context.kotlinBuildTargets
for (target in context.allTargets()) {
dataManager.getKotlinCache(target).clean()
dataManager.getKotlinCache(kotlinBuildTargets[target]!!).clean()
rebuildAfterCacheVersionChanged[target] = true
}
@@ -520,19 +501,14 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
}
}
private fun compilerArgumentsForChunk(chunk: ModuleChunk): CommonCompilerArguments =
chunk.representativeTarget().module.kotlinCompilerArguments
private fun doCompileModuleChunk(
allCompiledFiles: MutableSet<File>,
chunk: ModuleChunk,
kotlinTarget: KotlinModuleBuildTarget,
commonArguments: CommonCompilerArguments,
context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
roundDirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
environment: JpsCompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
incrementalCaches: Map<ModuleBuildTarget, IncrementalJvmCache>,
fsOperations: FSOperationsHelper
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
): OutputItemsCollector? {
val representativeTarget = chunk.representativeTarget()
@@ -556,39 +532,40 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
if (IncrementalCompilation.isEnabled()) {
for (target in chunk.targets) {
val cache = incrementalCaches[target]!!
val removedAndDirtyFiles = filesToCompile[target] + dirtyFilesHolder.getRemovedFiles(target).map(::File)
cache.markDirty(removedAndDirtyFiles)
val cache = incrementalCaches[target]
val targetDirtyFiles = roundDirtyFilesHolder.byTarget[target]
if (cache != null && targetDirtyFiles != null) {
val removedAndDirtyFiles: MutableSet<File> = mutableSetOf()
removedAndDirtyFiles.addAll(roundDirtyFilesHolder.getDirtyFiles(target))
removedAndDirtyFiles.addAll(roundDirtyFilesHolder.getRemovedFilesSet(target))
val complementaryFiles = cache.clearComplementaryFilesMapping(targetDirtyFiles.dirtyOrRemovedFiles)
targetDirtyFiles.markDirty(complementaryFiles)
cache.markDirty(targetDirtyFiles.dirtyOrRemovedFiles)
}
}
}
val kotlinModuleBuilderTarget = context.kotlinBuildTargets[representativeTarget]
val isDoneSomething = kotlinModuleBuilderTarget?.compileModuleChunk(
allCompiledFiles, chunk, commonArguments, dirtyFilesHolder,
environment, filesToCompile, fsOperations
) ?: false
val isDoneSomething = kotlinTarget.compileModuleChunk(
chunk, commonArguments, roundDirtyFilesHolder, environment
)
return if (isDoneSomething) environment.outputItemsCollector else null
}
private fun createCompileEnvironment(
incrementalCaches: Map<ModuleBuildTarget, IncrementalCache>,
kotlinModuleBuilderTarget: KotlinModuleBuildTarget,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>,
lookupTracker: LookupTracker,
context: CompileContext,
exceptActualTracer: ExpectActualTracker,
chunk: ModuleChunk,
messageCollector: MessageCollectorAdapter
): JpsCompilerEnvironment? {
val compilerServices = with(Services.Builder()) {
register(LookupTracker::class.java, lookupTracker)
register(
IncrementalCompilationComponents::class.java,
IncrementalCompilationComponentsImpl(incrementalCaches.mapKeys { context.kotlinBuildTargets[it.key]!!.targetId })
)
register(CompilationCanceledStatus::class.java, object : CompilationCanceledStatus {
override fun checkCanceled() {
if (context.cancelStatus.isCanceled) throw CompilationCanceledException()
}
})
kotlinModuleBuilderTarget.makeServices(this, incrementalCaches, lookupTracker, exceptActualTracer)
build()
}
@@ -607,7 +584,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
classesToLoadByParent,
messageCollector,
OutputItemsCollectorImpl(),
ProgressReporterImpl(context, chunk)
ProgressReporterImpl(kotlinModuleBuilderTarget.context, chunk)
)
}
@@ -641,7 +618,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val sourceToTarget = HashMap<File, ModuleBuildTarget>()
if (chunk.targets.size > 1) {
for (target in chunk.targets) {
context.kotlinBuildTargets[target]?.sources?.forEach {
context.kotlinBuildTargets[target]?.sourceFiles?.forEach {
sourceToTarget[it] = target
}
}
@@ -650,64 +627,14 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
val representativeTarget = chunk.representativeTarget()
fun SimpleOutputItem.target() =
sourceFiles.firstOrNull()?.let { sourceToTarget[it] } ?: chunk.targets.singleOrNull {
it.outputDir?.let {
outputFile.startsWith(it)
} ?: false
} ?: representativeTarget
it.outputDir?.let {
outputFile.startsWith(it)
} ?: false
} ?: representativeTarget
return outputItemCollector.outputs.groupBy(SimpleOutputItem::target, SimpleOutputItem::toGeneratedFile)
}
private fun updateJavaMappings(
chunk: ModuleChunk,
context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
outputItems: Map<ModuleBuildTarget, Iterable<GeneratedFile>>,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalJvmCache>
) {
val previousMappings = context.projectDescriptor.dataManager.mappings
val callback = JavaBuilderUtil.getDependenciesRegistrar(context)
val targetDirtyFiles: Map<ModuleBuildTarget, Set<File>> = chunk.targets.keysToMap {
val files = HashSet<File>()
dirtyFilesHolder.getRemovedFiles(it).mapTo(files, ::File)
files.addAll(filesToCompile.get(it))
files
}
fun getOldSourceFiles(target: ModuleBuildTarget, generatedClass: GeneratedJvmClass): Set<File> {
val cache = incrementalCaches[target] ?: return emptySet()
val className = generatedClass.outputClass.className
if (!cache.isMultifileFacade(className)) return emptySet()
val name = previousMappings.getName(className.internalName)
return previousMappings.getClassSources(name)?.toSet() ?: emptySet()
}
for ((target, outputs) in outputItems) {
for (output in outputs) {
if (output !is GeneratedJvmClass) continue
val sourceFiles = THashSet(FileUtil.FILE_HASHING_STRATEGY)
sourceFiles.addAll(getOldSourceFiles(target, output))
sourceFiles.removeAll(targetDirtyFiles[target] ?: emptySet())
sourceFiles.addAll(output.sourceFiles)
callback.associate(
FileUtil.toSystemIndependentName(output.outputFile.canonicalPath),
sourceFiles.map { FileUtil.toSystemIndependentName(it.canonicalPath) },
ClassReader(output.outputClass.fileContents)
)
}
}
val allCompiled = filesToCompile.values()
JavaBuilderUtil.registerFilesToCompile(context, allCompiled)
JavaBuilderUtil.registerSuccessfullyCompiled(context, allCompiled)
}
private fun registerOutputItems(outputConsumer: OutputConsumer, outputItems: Map<ModuleBuildTarget, List<GeneratedFile>>) {
for ((target, outputs) in outputItems) {
for (output in outputs) {
@@ -717,67 +644,18 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
}
private fun updateLookupStorage(
chunk: ModuleChunk,
lookupTracker: LookupTracker,
dataManager: BuildDataManager,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
filesToCompile: MultiMap<ModuleBuildTarget, File>
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder
) {
if (lookupTracker !is LookupTrackerImpl)
throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
val removedFiles = chunk.targets.flatMap { KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, it) }
dataManager.withLookupStorage { lookupStorage ->
lookupStorage.removeLookupsFrom(filesToCompile.values().asSequence() + removedFiles.asSequence())
lookupStorage.removeLookupsFrom(dirtyFilesHolder.dirtyOrRemovedFilesSet.asSequence())
lookupStorage.addAll(lookupTracker.lookups.entrySet(), lookupTracker.pathInterner.values)
}
}
class MessageCollectorAdapter(private val context: CompileContext) : MessageCollector {
private var hasErrors = false
override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
hasErrors = hasErrors or severity.isError
var prefix = ""
if (severity == EXCEPTION) {
prefix = INTERNAL_ERROR_PREFIX
}
val kind = kind(severity)
if (kind != null) {
context.processMessage(
CompilerMessage(
CompilerRunnerConstants.KOTLIN_COMPILER_NAME,
kind,
prefix + message,
location?.path,
-1, -1, -1,
location?.line?.toLong() ?: -1,
location?.column?.toLong() ?: -1
)
)
} else {
val path = if (location != null) "${location.path}:${location.line}:${location.column}: " else ""
KotlinBuilder.LOG.debug(path + message)
}
}
override fun clear() {
hasErrors = false
}
override fun hasErrors(): Boolean = hasErrors
private fun kind(severity: CompilerMessageSeverity): BuildMessage.Kind? {
return when (severity) {
INFO -> BuildMessage.Kind.INFO
ERROR, EXCEPTION -> BuildMessage.Kind.ERROR
WARNING, STRONG_WARNING -> BuildMessage.Kind.WARNING
LOGGING -> null
else -> throw IllegalArgumentException("Unsupported severity: $severity")
}
}
}
}
private class JpsICReporter : ICReporter {
@@ -792,7 +670,7 @@ private fun ChangesCollector.processChangesUsingLookups(
compiledFiles: Set<File>,
dataManager: BuildDataManager,
fsOperations: FSOperationsHelper,
caches: Iterable<IncrementalJvmCache>
caches: Iterable<JpsIncrementalCache>
) {
val allCaches = caches.flatMap { it.thisWithDependentCaches }
val reporter = JpsICReporter()
@@ -806,7 +684,7 @@ private fun ChangesCollector.processChangesUsingLookups(
}
private fun ChangesCollector.getDirtyFiles(
caches: Iterable<IncrementalCacheCommon<*>>,
caches: Iterable<IncrementalCacheCommon>,
dataManager: BuildDataManager
): Set<File> {
val reporter = JpsICReporter()
@@ -825,16 +703,32 @@ private fun getLookupTracker(project: JpsProject): LookupTracker {
return testLookupTracker
}
private fun getIncrementalCaches(chunk: ModuleChunk, context: CompileContext): Map<ModuleBuildTarget, JpsIncrementalJvmCache> {
private fun getIncrementalCaches(
chunk: ModuleChunk,
context: CompileContext
): Map<ModuleBuildTarget, JpsIncrementalCache> {
val dependentTargets = getDependentTargets(chunk, context)
val dataManager = context.projectDescriptor.dataManager
val chunkCaches = chunk.targets.keysToMap { dataManager.getKotlinCache(it) }
val dependentCaches = dependentTargets.map { dataManager.getKotlinCache(it) }
val kotlinBuildTargets = context.kotlinBuildTargets
val chunkCaches = chunk.targets.keysToMapExceptNulls {
val kotlinModuleBuilderTarget = kotlinBuildTargets[it]
if (kotlinModuleBuilderTarget !is KotlinCommonModuleBuildTarget) {
dataManager.getKotlinCache(kotlinBuildTargets[it]!!)
} else null
}
val dependentCaches = dependentTargets.mapNotNull {
val kotlinModuleBuilderTarget = kotlinBuildTargets[it]
if (kotlinModuleBuilderTarget !is KotlinCommonModuleBuildTarget) {
dataManager.getKotlinCache(kotlinModuleBuilderTarget!!)
} else null
}
for (chunkCache in chunkCaches.values) {
for (dependentCache in dependentCaches) {
chunkCache.addDependentCache(dependentCache)
chunkCache.addJpsDependentCache(dependentCache)
}
}
@@ -876,41 +770,5 @@ fun getDependentTargets(
private fun getDependenciesRecursively(module: JpsModule, kind: JpsJavaClasspathKind): Set<JpsModule> =
JpsJavaExtensionService.dependencies(module).includedIn(kind).recursivelyExportedOnly().modules
// TODO: investigate thread safety
private val ALL_COMPILED_FILES_KEY = Key.create<MutableSet<File>>("_all_kotlin_compiled_files_")
fun getAllCompiledFilesContainer(context: CompileContext): MutableSet<File> {
var allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context)
if (allCompiledFiles == null) {
allCompiledFiles = THashSet(FileUtil.FILE_HASHING_STRATEGY)
ALL_COMPILED_FILES_KEY.set(context, allCompiledFiles)
}
return allCompiledFiles
}
// TODO: investigate thread safety
private val PROCESSED_TARGETS_WITH_REMOVED_FILES = Key.create<MutableSet<ModuleBuildTarget>>("_processed_targets_with_removed_files_")
val CompileContext.processedTargetsWithRemovedFilesContainer: MutableSet<ModuleBuildTarget>
get() {
var set = PROCESSED_TARGETS_WITH_REMOVED_FILES.get(this)
if (set == null) {
set = HashSet()
PROCESSED_TARGETS_WITH_REMOVED_FILES.set(this, set)
}
return set
}
private fun hasKotlinDirtyOrRemovedFiles(
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
chunk: ModuleChunk
): Boolean {
if (!dirtyFilesHolder.hasDirtyFiles() && !dirtyFilesHolder.hasRemovedFiles()) return false
if (!KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder).isEmpty) return true
return chunk.targets.any { KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, it).isNotEmpty() }
}
fun jvmBuildMetaInfoFile(target: ModuleBuildTarget, dataManager: BuildDataManager): File =
File(dataManager.dataPaths.getTargetDataRoot(target), KotlinBuilder.JVM_BUILD_META_INFO_FILE_NAME)
@@ -0,0 +1,86 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.build
import org.jetbrains.jps.builders.AdditionalRootsProviderService
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.model.java.JavaSourceRootProperties
import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.model.module.JpsModuleSourceRootType
import org.jetbrains.kotlin.config.KotlinSourceRootType
import org.jetbrains.kotlin.jps.model.expectedByModules
import java.io.File
/**
* Required for Multiplatform Projects.
*
* Adds all the source roots of the expectedBy modules to the platform modules.
*/
class KotlinMppCommonSourceRootProvider : AdditionalRootsProviderService<JavaSourceRootDescriptor>(JavaModuleBuildTargetType.ALL_TYPES) {
override fun getAdditionalRoots(
target: BuildTarget<JavaSourceRootDescriptor>,
dataPaths: BuildDataPaths?
): List<JavaSourceRootDescriptor> {
val moduleBuildTarget = target as? ModuleBuildTarget ?: return listOf()
val module = moduleBuildTarget.module
val result = mutableListOf<JavaSourceRootDescriptor>()
module.expectedByModules.forEach { commonModule ->
addSourceRoots(result, commonModule, target)
}
return result
}
private fun addSourceRoots(
result: MutableList<JavaSourceRootDescriptor>,
commonModule: JpsModule,
target: ModuleBuildTarget
) {
for (commonSourceRoot in commonModule.sourceRoots) {
val isCommonTestsRootType = commonSourceRoot.rootType.isTestsRootType
if (isCommonTestsRootType == null || target.isTests == isCommonTestsRootType) {
val javaSourceRootProperties = commonSourceRoot.properties as? JavaSourceRootProperties
result.add(
KotlinCommonModuleSourceRoot(
commonModule,
commonSourceRoot.file,
target,
javaSourceRootProperties?.isForGeneratedSources ?: false,
false,
javaSourceRootProperties?.packagePrefix ?: "",
setOf()
)
)
}
}
}
}
private val JpsModuleSourceRootType<*>.isTestsRootType
get() = when (this) {
is KotlinSourceRootType -> this == KotlinSourceRootType.TestSource
is JavaSourceRootType -> this == JavaSourceRootType.TEST_SOURCE // for compatibility
else -> null
}
class KotlinCommonModuleSourceRoot(
val commonModule: JpsModule,
root: File,
target: ModuleBuildTarget,
isGenerated: Boolean,
isTemp: Boolean,
packagePrefix: String,
excludes: Set<File>
) : JavaSourceRootDescriptor(root, target, isGenerated, isTemp, packagePrefix, excludes)
@@ -0,0 +1,141 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.build
import com.intellij.openapi.util.io.FileUtilRt
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.FileProcessor
import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
import java.io.File
/**
* Holding kotlin dirty files list for particular build round.
* Dirty and removed files set are initialized from [delegate].
*
* Probably should be merged with [FSOperationsHelper]
*/
class KotlinRoundDirtySourceFilesHolder(
val chunk: ModuleChunk,
val context: CompileContext,
val fsOperations: FSOperationsHelper,
delegate: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>
) : DirtyFilesHolderBase<JavaSourceRootDescriptor, ModuleBuildTarget>(context) {
val byTarget: Map<ModuleBuildTarget, TargetFiles>
inner class TargetFiles(val target: ModuleBuildTarget, val removed: Collection<String>) {
val dirty: MutableSet<DirtyFile> = mutableSetOf()
val dirtyOrRemovedFiles: Set<File>
get() {
val result = mutableSetOf<File>()
dirty.forEach { result.add(it.file) }
removed.forEach { result.add(File(it)) }
return result
}
fun markDirty(files: Collection<File>) {
fsOperations.markFiles(files)
files.forEach {
dirty.add(DirtyFile(it, null))
}
}
}
data class DirtyFile(val file: File, val root: JavaSourceRootDescriptor?)
val hasRemovedFiles: Boolean
override fun hasRemovedFiles(): Boolean = hasRemovedFiles
val hasDirtyFiles: Boolean
override fun hasDirtyFiles(): Boolean = hasDirtyFiles
val hasDirtyOrRemovedFiles: Boolean
get() = hasRemovedFiles || hasDirtyFiles
init {
val byTarget = mutableMapOf<ModuleBuildTarget, TargetFiles>()
var hasDirtyFiles = false
var hasRemovedFiles = false
chunk.targets.forEach { target ->
val removedFiles = delegate.getRemovedFiles(target)
if (removedFiles.isNotEmpty()) {
hasRemovedFiles = true
}
byTarget[target] = TargetFiles(target, removedFiles)
}
delegate.processDirtyFiles { target, file, root ->
val targetInfo = byTarget[target]
?: error("processDirtyFiles should callback only on chunk `$chunk` targets (`$target` is not)")
if (file.isKotlinSourceFile) {
targetInfo.dirty.add(DirtyFile(file, root))
hasDirtyFiles = true
}
true
}
this.byTarget = byTarget
this.hasRemovedFiles = hasRemovedFiles
this.hasDirtyFiles = hasDirtyFiles
}
override fun processDirtyFiles(processor: FileProcessor<JavaSourceRootDescriptor, ModuleBuildTarget>) {
for ((target, info) in byTarget) {
info.dirty.forEach {
if (!processor.apply(target, it.file, it.root)) return
}
}
}
fun getDirtyFiles(target: ModuleBuildTarget): Set<File> =
byTarget[target]?.dirty?.mapTo(mutableSetOf()) { it.file } ?: setOf()
fun getRemovedFilesSet(target: ModuleBuildTarget): Set<File> =
byTarget[target]?.removed?.mapTo(mutableSetOf()) { File(it) } ?: setOf()
override fun getRemovedFiles(target: ModuleBuildTarget): Collection<String> =
byTarget.flatMap { it.value.removed }
val removedFilesCount
get() = byTarget.values.flatMapTo(mutableSetOf()) { it.removed }.size
val dirtyFiles: Set<File>
get() = byTarget.flatMapTo(mutableSetOf()) { it.value.dirty.map { it.file } }
val dirtyOrRemovedFilesSet: Set<File>
get() {
val result = mutableSetOf<File>()
byTarget.forEach {
it.value.dirty.forEach { result.add(it.file) }
it.value.removed.forEach { result.add(File(it)) }
}
return result
}
}
val File.isKotlinSourceFile: Boolean
get() = FileUtilRt.extensionEquals(name, "kt")
@@ -1,53 +0,0 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.build
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.util.containers.MultiMap
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.incremental.ModuleBuildTarget
import java.io.File
object KotlinSourceFileCollector {
// For incremental compilation
fun getDirtySourceFiles(dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>): MultiMap<ModuleBuildTarget, File> {
val result = MultiMap<ModuleBuildTarget, File>()
dirtyFilesHolder.processDirtyFiles { target, file, root ->
//TODO this is a workaround for bug in JPS: the latter erroneously calls process with invalid parameters
if (root.getTarget() == target && isKotlinSourceFile(file)) {
result.putValue(target, file)
}
true
}
return result
}
fun getRemovedKotlinFiles(
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
target: ModuleBuildTarget
): List<File> =
dirtyFilesHolder
.getRemovedFiles(target)
.mapNotNull { if (FileUtilRt.extensionEquals(it, "kt")) File(it) else null }
internal fun isKotlinSourceFile(file: File): Boolean {
return FileUtilRt.extensionEquals(file.name, "kt")
}
}
@@ -0,0 +1,71 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.build
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.messages.BuildMessage
import org.jetbrains.jps.incremental.messages.CompilerMessage
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.config.CompilerRunnerConstants
import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget
class MessageCollectorAdapter(
private val context: CompileContext,
val kotlinTarget: KotlinModuleBuildTarget?
) : MessageCollector {
private var hasErrors = false
override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
hasErrors = hasErrors || severity.isError
var prefix = ""
if (severity == CompilerMessageSeverity.EXCEPTION) {
prefix = CompilerRunnerConstants.INTERNAL_ERROR_PREFIX
}
val kind = kind(severity)
if (kind != null) {
// Report target when cross-compiling common files
if (location != null && kotlinTarget != null && kotlinTarget.isCommonModuleFile(location.path)) {
val moduleName = kotlinTarget.module.name
prefix += "[$moduleName] "
}
context.processMessage(
CompilerMessage(
CompilerRunnerConstants.KOTLIN_COMPILER_NAME,
kind,
prefix + message,
location?.path,
-1, -1, -1,
location?.line?.toLong() ?: -1,
location?.column?.toLong() ?: -1
)
)
} else {
val path = if (location != null) "${location.path}:${location.line}:${location.column}: " else ""
KotlinBuilder.LOG.debug(path + message)
}
}
override fun clear() {
hasErrors = false
}
override fun hasErrors(): Boolean = hasErrors
private fun kind(severity: CompilerMessageSeverity): BuildMessage.Kind? {
return when (severity) {
CompilerMessageSeverity.INFO -> BuildMessage.Kind.INFO
CompilerMessageSeverity.ERROR, CompilerMessageSeverity.EXCEPTION -> BuildMessage.Kind.ERROR
CompilerMessageSeverity.WARNING, CompilerMessageSeverity.STRONG_WARNING -> BuildMessage.Kind.WARNING
CompilerMessageSeverity.LOGGING -> null
else -> throw IllegalArgumentException("Unsupported severity: $severity")
}
}
}
@@ -21,32 +21,50 @@ import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.kotlin.incremental.IncrementalCacheCommon
import org.jetbrains.kotlin.incremental.IncrementalJsCache
import org.jetbrains.kotlin.incremental.IncrementalJvmCache
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget
import java.io.File
interface JpsIncrementalCache : IncrementalCacheCommon, StorageOwner {
fun addJpsDependentCache(cache: JpsIncrementalCache)
}
class JpsIncrementalJvmCache(
target: ModuleBuildTarget,
paths: BuildDataPaths
) : IncrementalJvmCache(paths.getTargetDataRoot(target), target.outputDir), StorageOwner {
) : IncrementalJvmCache(paths.getTargetDataRoot(target), target.outputDir), JpsIncrementalCache {
override fun addJpsDependentCache(cache: JpsIncrementalCache) {
addDependentCache(cache as JpsIncrementalJvmCache)
}
override fun debugLog(message: String) {
KotlinBuilder.LOG.debug(message)
}
}
private class KotlinIncrementalStorageProvider(
private val target: ModuleBuildTarget,
private val paths: BuildDataPaths
) : StorageProvider<JpsIncrementalJvmCache>() {
class JpsIncrementalJsCache(
target: ModuleBuildTarget,
paths: BuildDataPaths
) : IncrementalJsCache(paths.getTargetDataRoot(target)), JpsIncrementalCache {
override fun addJpsDependentCache(cache: JpsIncrementalCache) {
addDependentCache(cache as JpsIncrementalJsCache)
}
}
private class KotlinIncrementalStorageProvider(
private val target: KotlinModuleBuildTarget,
private val paths: BuildDataPaths
) : StorageProvider<JpsIncrementalCache>() {
override fun equals(other: Any?) = other is KotlinIncrementalStorageProvider && target == other.target
override fun hashCode() = target.hashCode()
override fun createStorage(targetDataDir: File): JpsIncrementalJvmCache =
JpsIncrementalJvmCache(target, paths)
override fun createStorage(targetDataDir: File): JpsIncrementalCache = target.createCacheStorage(paths)
}
fun BuildDataManager.getKotlinCache(target: ModuleBuildTarget): JpsIncrementalJvmCache =
getStorage(target, KotlinIncrementalStorageProvider(target, dataPaths))
fun BuildDataManager.getKotlinCache(target: KotlinModuleBuildTarget): JpsIncrementalCache =
getStorage(target.jpsModuleBuildTarget, KotlinIncrementalStorageProvider(target, dataPaths))
@@ -5,8 +5,11 @@
package org.jetbrains.kotlin.jps.model
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.model.ex.JpsElementBase
import org.jetbrains.jps.model.ex.JpsElementChildRoleBase
import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.kotlin.cli.common.arguments.*
import org.jetbrains.kotlin.config.CompilerSettings
@@ -19,6 +22,25 @@ val JpsModule.kotlinFacet: JpsKotlinFacetModuleExtension?
val JpsModule.targetPlatform: TargetPlatformKind<*>?
get() = kotlinFacet?.settings?.targetPlatformKind
val JpsModule.expectedByModules: List<JpsModule>
get() {
val kotlinFacetExtension = kotlinFacet
val implementedModuleNames = kotlinFacetExtension?.settings?.implementedModuleNames ?: return listOf()
if (implementedModuleNames.isEmpty()) return listOf()
val result = ArrayList<JpsModule>(implementedModuleNames.size)
JpsJavaExtensionService.dependencies(this)
.processModules {
if (it.name in implementedModuleNames) {
// Note, production sources should be added for both production and tests targets
result.add(it)
}
}
return result
}
val JpsModule.productionOutputFilePath: String?
get() {
val facetSettings = kotlinFacet?.settings ?: return null
@@ -59,7 +81,7 @@ private inline fun <reified T : CommonCompilerArguments> JpsModule.getCompilerAr
val facetSettings = kotlinFacet?.settings ?: return projectSettingsCopy
if (facetSettings.useProjectSettings) return projectSettingsCopy
return facetSettings.compilerArguments as? T ?: projectSettingsCopy
return facetSettings.mergedCompilerArguments as? T ?: projectSettingsCopy
}
class JpsKotlinFacetModuleExtension(settings: KotlinFacetSettings) : JpsElementBase<JpsKotlinFacetModuleExtension>() {
@@ -5,49 +5,87 @@
package org.jetbrains.kotlin.jps.platforms
import com.intellij.util.containers.MultiMap
import com.intellij.util.xmlb.XmlSerializerUtil
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import com.intellij.openapi.compiler.CompileContext as JpsCompileContext
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.FSOperations
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.fs.CompilationRound
import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.mergeBeans
import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner
import org.jetbrains.kotlin.jps.build.FSOperationsHelper
import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
import org.jetbrains.kotlin.jps.model.k2MetadataCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import java.io.File
class KotlinCommonModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) :
KotlinModuleBuilderTarget(compileContext, jpsModuleBuildTarget) {
class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) :
KotlinModuleBuildTarget(context, jpsModuleBuildTarget) {
override fun compileModuleChunk(
allCompiledFiles: MutableSet<File>,
chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
environment: JpsCompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
fsOperations: FSOperationsHelper
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
): Boolean {
require(chunk.representativeTarget() == jpsModuleBuildTarget)
if (reportAndSkipCircular(chunk, environment)) return false
reportAndSkipCircular(chunk, environment)
// Incremental compilation is not supported, so mark all dependents as dirty
FSOperations.markDirtyRecursively(context, CompilationRound.CURRENT, chunk)
JpsKotlinCompilerRunner().runK2MetadataCompiler(
commonArguments,
module.k2MetadataCompilerArguments,
module.kotlinCompilerSettings,
environment,
sources
)
val k2MetadataArguments = module.k2MetadataCompilerArguments
if (k2MetadataArguments.enabledInJps || (commonArguments as? K2MetadataCompilerArguments)?.enabledInJps == true) {
JpsKotlinCompilerRunner().runK2MetadataCompiler(
commonArguments,
k2MetadataArguments,
module.kotlinCompilerSettings,
environment,
destination,
dependenciesOutputDirs + libraryFiles,
sourceFiles // incremental K2MetadataCompiler not supported yet
)
}
return true
}
private val libraryFiles: List<String>
get() = mutableListOf<String>().also { result ->
for (library in allDependencies.libraries) {
for (root in library.getRoots(JpsOrderRootType.COMPILED)) {
result.add(JpsPathUtil.urlToPath(root.url))
}
}
}
private val dependenciesOutputDirs: List<String>
get() = mutableListOf<String>().also { result ->
allDependencies.processModules { module ->
if (isTests) addDependencyMetaFile(module, result, isTests = true)
// note: production targets should be also added as dependency to test targets
addDependencyMetaFile(module, result, isTests = false)
}
}
val destination: String
get() = module.k2MetadataCompilerArguments.destination ?: outputDir.absolutePath
private fun addDependencyMetaFile(
module: JpsModule,
result: MutableList<String>,
isTests: Boolean
) {
val dependencyBuildTarget = context.kotlinBuildTargets[ModuleBuildTarget(module, isTests)]
if (dependencyBuildTarget != this@KotlinCommonModuleBuildTarget &&
dependencyBuildTarget is KotlinCommonModuleBuildTarget &&
dependencyBuildTarget.sourceFiles.isNotEmpty()
) {
result.add(dependencyBuildTarget.destination)
}
}
override fun createCacheStorage(paths: BuildDataPaths) = JpsIncrementalJvmCache(jpsModuleBuildTarget, paths) // todo: delete
}
@@ -5,19 +5,30 @@
package org.jetbrains.kotlin.jps.platforms
import com.intellij.util.containers.MultiMap
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner
import org.jetbrains.kotlin.jps.build.FSOperationsHelper
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.ChangesCollector
import org.jetbrains.kotlin.incremental.IncrementalJsCache
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.js.TranslationResultValue
import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJsCache
import org.jetbrains.kotlin.jps.model.k2JsCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
@@ -28,23 +39,56 @@ import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.META_JS_SUFFIX
import java.io.File
import java.net.URI
internal class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : IncrementalDataProvider {
override val headerMetadata: ByteArray
get() = cache.header
override val compiledPackageParts: Map<File, TranslationResultValue>
get() = cache.nonDirtyPackageParts()
}
class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) :
KotlinModuleBuilderTarget(compileContext, jpsModuleBuildTarget) {
KotlinModuleBuildTarget(compileContext, jpsModuleBuildTarget) {
val sourceToOutputMap
get() = context.projectDescriptor.dataManager.getSourceToOutputMap(jpsModuleBuildTarget)
val isFirstBuild
get() = sourceToOutputMap.sources.isEmpty()
override fun makeServices(
builder: Services.Builder,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>,
lookupTracker: LookupTracker,
exceptActualTracer: ExpectActualTracker
) {
super.makeServices(builder, incrementalCaches, lookupTracker, exceptActualTracer)
with(builder) {
register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl())
if (IncrementalCompilation.isEnabled() && !isFirstBuild) {
val cache = incrementalCaches[jpsModuleBuildTarget] as IncrementalJsCache
register(
IncrementalDataProvider::class.java,
IncrementalDataProviderFromCache(cache)
)
}
}
}
override fun compileModuleChunk(
allCompiledFiles: MutableSet<File>,
chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
environment: JpsCompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
fsOperations: FSOperationsHelper
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
): Boolean {
require(chunk.representativeTarget() == jpsModuleBuildTarget)
if (reportAndSkipCircular(chunk, environment)) return false
val sources = sources
if (sources.isEmpty()) return false
val sources = collectSourcesToCompile(dirtyFilesHolder)
if (!checkShouldCompileAndLog(dirtyFilesHolder, sources)) return false
val libraries = libraryFiles + dependenciesMetaFiles
@@ -144,7 +188,7 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa
if (dependencyBuildTarget != this@KotlinJsModuleBuildTarget &&
dependencyBuildTarget is KotlinJsModuleBuildTarget &&
dependencyBuildTarget.sources.isNotEmpty()
dependencyBuildTarget.sourceFiles.isNotEmpty()
) {
val metaFile = dependencyBuildTarget.outputMetaFile
if (metaFile.exists()) {
@@ -152,4 +196,24 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa
}
}
}
override fun createCacheStorage(paths: BuildDataPaths) =
JpsIncrementalJsCache(jpsModuleBuildTarget, paths)
override fun updateCaches(
jpsIncrementalCache: JpsIncrementalCache,
files: List<GeneratedFile>,
changesCollector: ChangesCollector,
environment: JpsCompilerEnvironment
) {
super.updateCaches(jpsIncrementalCache, files, changesCollector, environment)
val incrementalResults = environment.services[IncrementalResultsConsumer::class.java] as IncrementalResultsConsumerImpl
val jsCache = jpsIncrementalCache as IncrementalJsCache
jsCache.header = incrementalResults.headerMetadata
jsCache.compareAndUpdate(incrementalResults, changesCollector)
jsCache.clearCacheForRemovedClasses(changesCollector)
}
}
@@ -9,42 +9,120 @@ import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.text.StringUtil
import com.intellij.openapi.vfs.StandardFileSystems
import com.intellij.util.containers.ContainerUtil
import com.intellij.util.containers.MultiMap
import com.intellij.util.io.URLUtil
import gnu.trove.THashSet
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.builders.java.JavaBuilderUtil
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsSdkDependency
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.JvmBuildMetaInfo
import org.jetbrains.kotlin.build.JvmSourceRoot
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.jps.build.FSOperationsHelper
import org.jetbrains.kotlin.config.ApiVersion
import org.jetbrains.kotlin.config.LanguageVersion
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.jps.build.KotlinSourceFileCollector
import org.jetbrains.kotlin.jps.build.processedTargetsWithRemovedFilesContainer
import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
import org.jetbrains.kotlin.jps.build.jvmBuildMetaInfoFile
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
import org.jetbrains.kotlin.jps.model.k2JvmCompilerArguments
import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.KotlinModuleXmlBuilder
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.utils.keysToMap
import org.jetbrains.org.objectweb.asm.ClassReader
import java.io.File
import java.io.IOException
class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) :
KotlinModuleBuilderTarget(compileContext, jpsModuleBuildTarget) {
KotlinModuleBuildTarget(compileContext, jpsModuleBuildTarget) {
override fun createCacheStorage(paths: BuildDataPaths) = JpsIncrementalJvmCache(jpsModuleBuildTarget, paths)
override fun checkCachesVersions(chunk: ModuleChunk, dataManager: BuildDataManager, actions: MutableSet<CacheVersion.Action>) {
val args = compilerArgumentsForChunk(chunk)
val currentBuildMetaInfo = JvmBuildMetaInfo(args)
for (target in chunk.targets) {
val file = jvmBuildMetaInfoFile(target, dataManager)
if (!file.exists()) continue
val lastBuildMetaInfo =
try {
JvmBuildMetaInfo.deserializeFromString(file.readText()) ?: continue
} catch (e: Exception) {
KotlinBuilder.LOG.error("Could not deserialize jvm build meta info", e)
continue
}
val lastBuildLangVersion = LanguageVersion.fromVersionString(lastBuildMetaInfo.languageVersionString)
val lastBuildApiVersion = ApiVersion.parse(lastBuildMetaInfo.apiVersionString)
val currentLangVersion =
args.languageVersion?.let { LanguageVersion.fromVersionString(it) } ?: LanguageVersion.LATEST_STABLE
val currentApiVersion =
args.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.createByLanguageVersion(currentLangVersion)
val reasonToRebuild = when {
currentLangVersion != lastBuildLangVersion -> {
"Language version was changed ($lastBuildLangVersion -> $currentLangVersion)"
}
currentApiVersion != lastBuildApiVersion -> {
"Api version was changed ($lastBuildApiVersion -> $currentApiVersion)"
}
lastBuildLangVersion != LanguageVersion.KOTLIN_1_0 && lastBuildMetaInfo.isEAP && !currentBuildMetaInfo.isEAP -> {
// If EAP->Non-EAP build with IC, then rebuild all kotlin
"Last build was compiled with EAP-plugin"
}
else -> null
}
if (reasonToRebuild != null) {
KotlinBuilder.LOG.info("$reasonToRebuild. Performing non-incremental rebuild (kotlin only)")
actions.add(CacheVersion.Action.REBUILD_ALL_KOTLIN)
}
}
}
override fun makeServices(
builder: Services.Builder,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>,
lookupTracker: LookupTracker,
exceptActualTracer: ExpectActualTracker
) {
super.makeServices(builder, incrementalCaches, lookupTracker, exceptActualTracer)
with(builder) {
register(
IncrementalCompilationComponents::class.java,
IncrementalCompilationComponentsImpl(
incrementalCaches.mapKeys { context.kotlinBuildTargets[it.key]!!.targetId } as Map<TargetId, IncrementalCache>
)
)
}
}
override fun compileModuleChunk(
allCompiledFiles: MutableSet<File>,
chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
environment: JpsCompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
fsOperations: FSOperationsHelper
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
): Boolean {
if (chunk.modules.size > 1) {
environment.messageCollector.report(
@@ -55,34 +133,28 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
)
}
allCompiledFiles.addAll(filesToCompile.values())
val filesSet = dirtyFilesHolder.dirtyFiles
val processedTargetsWithRemoved = context.processedTargetsWithRemovedFilesContainer
var totalRemovedFiles = 0
for (target in chunk.targets) {
val removedFilesInTarget = KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target)
if (!removedFilesInTarget.isEmpty()) {
if (processedTargetsWithRemoved.add(target)) {
totalRemovedFiles += removedFilesInTarget.size
}
}
}
val moduleFile = generateModuleDescription(context, chunk, filesToCompile, totalRemovedFiles != 0)
val moduleFile = generateModuleDescription(chunk, dirtyFilesHolder)
if (moduleFile == null) {
KotlinBuilder.LOG.debug(
"Not compiling, because no files affected: " + filesToCompile.keySet().joinToString { it.presentableName }
)
if (KotlinBuilder.LOG.isDebugEnabled) {
KotlinBuilder.LOG.debug(
"Not compiling, because no files affected: " + chunk.targets.joinToString { it.presentableName }
)
}
// No Kotlin sources found
return false
}
val module = chunk.representativeTarget().module
KotlinBuilder.LOG.debug("Compiling to JVM ${filesToCompile.values().size} files"
+ (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)")
+ " in " + filesToCompile.keySet().joinToString { it.presentableName })
if (KotlinBuilder.LOG.isDebugEnabled) {
val totalRemovedFiles = dirtyFilesHolder.removedFilesCount
KotlinBuilder.LOG.debug("Compiling to JVM ${filesSet.size} files"
+ (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)")
+ " in " + chunk.targets.joinToString { it.presentableName })
}
try {
val compilerRunner = JpsKotlinCompilerRunner()
@@ -102,39 +174,22 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
return true
}
fun generateModuleDescription(
context: CompileContext,
chunk: ModuleChunk,
sourceFiles: MultiMap<ModuleBuildTarget, File>, // ignored for non-incremental compilation
hasRemovedFiles: Boolean
): File? {
fun generateModuleDescription(chunk: ModuleChunk, dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder): File? {
val builder = KotlinModuleXmlBuilder()
var noSources = true
var hasDirtySources = false
val targets = chunk.targets.mapNotNull { this.context.kotlinBuildTargets[it] as? KotlinJvmModuleBuildTarget }
val outputDirs = targets.map { it.outputDir }.toSet()
val logger = context.loggingManager.projectBuilderLogger
for (target in targets) {
val outputDir = target.outputDir
val friendDirs = target.friendOutputDirs
val moduleSources =
if (IncrementalCompilation.isEnabled() && target.expectedBy.isEmpty()) {
sourceFiles.get(target.jpsModuleBuildTarget)
} else {
target.sources
}
if (moduleSources.isNotEmpty() || hasRemovedFiles) {
noSources = false
if (logger.isEnabled) {
logger.logCompiledFiles(moduleSources, KotlinBuilder.KOTLIN_BUILDER_NAME, "Compiling files:")
}
}
val moduleSources = collectSourcesToCompile(target, dirtyFilesHolder)
val hasDirtyOrRemovedSources = checkShouldCompileAndLog(target, dirtyFilesHolder, moduleSources)
if (hasDirtyOrRemovedSources) hasDirtySources = true
val kotlinModuleId = target.targetId
builder.addModule(
@@ -152,7 +207,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
)
}
if (noSources) return null
if (!hasDirtySources) return null
val scriptFile = createTempFileForModuleDesc(chunk)
FileUtil.writeToFile(scriptFile, builder.asText().toString())
@@ -192,7 +247,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
val extension = file.extension
// Don't filter out files, we want to report warnings about absence through the common place
if (!(extension == "class" || extension == "jar")) {
if (extension != "class" && extension != "jar") {
return@filter false
}
}
@@ -213,7 +268,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
return File(url.substringAfter(StandardFileSystems.JRT_PROTOCOL_PREFIX).substringBeforeLast(URLUtil.JAR_SEPARATOR))
}
fun findSourceRoots(context: CompileContext): List<JvmSourceRoot> {
private fun findSourceRoots(context: CompileContext): List<JvmSourceRoot> {
val roots = context.projectDescriptor.buildRootIndex.getTargetRoots(jpsModuleBuildTarget, context)
val result = ContainerUtil.newArrayList<JvmSourceRoot>()
for (root in roots) {
@@ -225,4 +280,64 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
}
return result
}
override fun updateCaches(
jpsIncrementalCache: JpsIncrementalCache,
files: List<GeneratedFile>,
changesCollector: ChangesCollector,
environment: JpsCompilerEnvironment
) {
super.updateCaches(jpsIncrementalCache, files, changesCollector, environment)
updateIncrementalCache(files, jpsIncrementalCache as IncrementalJvmCache, changesCollector, null)
}
override fun updateChunkMappings(
chunk: ModuleChunk,
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
outputItems: Map<ModuleBuildTarget, Iterable<GeneratedFile>>,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
) {
val previousMappings = context.projectDescriptor.dataManager.mappings
val callback = JavaBuilderUtil.getDependenciesRegistrar(context)
val targetDirtyFiles: Map<ModuleBuildTarget, Set<File>> = chunk.targets.keysToMap {
val files = HashSet<File>()
dirtyFilesHolder.getRemovedFiles(it).mapTo(files, ::File)
files.addAll(dirtyFilesHolder.getDirtyFiles(it))
files
}
fun getOldSourceFiles(target: ModuleBuildTarget, generatedClass: GeneratedJvmClass): Set<File> {
val cache = incrementalCaches[target] ?: return emptySet()
cache as JpsIncrementalJvmCache
val className = generatedClass.outputClass.className
if (!cache.isMultifileFacade(className)) return emptySet()
val name = previousMappings.getName(className.internalName)
return previousMappings.getClassSources(name)?.toSet() ?: emptySet()
}
for ((target, outputs) in outputItems) {
for (output in outputs) {
if (output !is GeneratedJvmClass) continue
val sourceFiles = THashSet(FileUtil.FILE_HASHING_STRATEGY)
sourceFiles.addAll(getOldSourceFiles(target, output))
sourceFiles.removeAll(targetDirtyFiles[target] ?: emptySet())
sourceFiles.addAll(output.sourceFiles)
callback.associate(
FileUtil.toSystemIndependentName(output.outputFile.canonicalPath),
sourceFiles.map { FileUtil.toSystemIndependentName(it.canonicalPath) },
ClassReader(output.outputClass.fileContents)
)
}
}
val allCompiled = dirtyFilesHolder.dirtyFiles
JavaBuilderUtil.registerFilesToCompile(context, allCompiled)
JavaBuilderUtil.registerSuccessfullyCompiled(context, allCompiled)
}
}
@@ -0,0 +1,263 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.platforms
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.ProjectBuildException
import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.CacheVersion
import org.jetbrains.kotlin.incremental.ChangesCollector
import org.jetbrains.kotlin.incremental.ExpectActualTrackerImpl
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.jps.build.*
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.progress.CompilationCanceledException
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
import org.jetbrains.kotlin.utils.addIfNotNull
import java.io.File
/**
* Properties and actions for Kotlin test / production module build target.
*/
abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModuleBuildTarget: ModuleBuildTarget) {
val module: JpsModule
get() = jpsModuleBuildTarget.module
val isTests: Boolean
get() = jpsModuleBuildTarget.isTests
val targetId: TargetId
get() {
// Since IDEA 2016 each gradle source root is imported as a separate module.
// One gradle module X is imported as two JPS modules:
// 1. X-production with one production target;
// 2. X-test with one test target.
// This breaks kotlin code since internal members' names are mangled using module name.
// For example, a declaration of a function 'f' in 'X-production' becomes 'fXProduction', but a call 'f' in 'X-test' becomes 'fXTest()'.
// The workaround is to replace a name of such test target with the name of corresponding production module.
// See KT-11993.
val name = relatedProductionModule?.name ?: jpsModuleBuildTarget.id
return TargetId(name, jpsModuleBuildTarget.targetType.typeId)
}
val outputDir by lazy {
val explicitOutputPath = if (isTests) module.testOutputFilePath else module.productionOutputFilePath
val explicitOutputDir = explicitOutputPath?.let { File(it).absoluteFile.parentFile }
return@lazy explicitOutputDir
?: jpsModuleBuildTarget.outputDir
?: throw ProjectBuildException("No output directory found for " + this)
}
val friendBuildTargets: List<KotlinModuleBuildTarget>
get() {
val result = mutableListOf<KotlinModuleBuildTarget>()
if (isTests) {
result.addIfNotNull(context.kotlinBuildTargets[module.productionBuildTarget])
result.addIfNotNull(context.kotlinBuildTargets[relatedProductionModule?.productionBuildTarget])
}
return result.filter { it.sourceFiles.isNotEmpty() }
}
val friendOutputDirs: List<File>
get() = friendBuildTargets.mapNotNull {
JpsJavaExtensionService.getInstance().getOutputDirectory(it.module, false)
}
private val relatedProductionModule: JpsModule?
get() = JpsJavaExtensionService.getInstance().getTestModuleProperties(module)?.productionModule
val allDependencies by lazy {
JpsJavaExtensionService.dependencies(module).recursively().exportedOnly()
.includedIn(JpsJavaClasspathKind.compile(isTests))
}
val sources by lazy {
mutableMapOf<String, Source>().also { result ->
collectSources(result)
}
}
private fun collectSources(receiver: MutableMap<String, Source>) {
val moduleExcludes = module.excludeRootsList.urls.mapTo(java.util.HashSet(), JpsPathUtil::urlToFile)
val compilerExcludes = JpsJavaExtensionService.getInstance()
.getOrCreateCompilerConfiguration(module.project)
.compilerExcludes
val buildRootIndex = context.projectDescriptor.buildRootIndex
val roots = buildRootIndex.getTargetRoots(jpsModuleBuildTarget, context)
roots.forEach { rootDescriptor ->
val isCommonRoot = rootDescriptor is KotlinCommonModuleSourceRoot
rootDescriptor.root.walkTopDown()
.onEnter { file -> file !in moduleExcludes }
.forEach { file ->
if (!compilerExcludes.isExcluded(file) && file.isFile && file.isKotlinSourceFile) {
receiver[file.path] = Source(file, isCommonRoot)
}
}
}
}
/**
* @property isCommonModule for reporting errors during cross-compilation common module sources
*/
class Source(
val file: File,
val isCommonModule: Boolean
)
fun isCommonModuleFile(path: String): Boolean = sources[path]?.isCommonModule == true
val sourceFiles by lazy {
sources.values.map { it.file }
}
override fun toString() = jpsModuleBuildTarget.toString()
/**
* Called for `ModuleChunk.representativeTarget`
*/
abstract fun compileModuleChunk(
chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
environment: JpsCompilerEnvironment
): Boolean
protected fun reportAndSkipCircular(
chunk: ModuleChunk,
environment: JpsCompilerEnvironment
): Boolean {
if (chunk.modules.size > 1) {
// We do not support circular dependencies, but if they are present, we do our best should not break the build,
// so we simply yield a warning and report NOTHING_DONE
environment.messageCollector.report(
CompilerMessageSeverity.STRONG_WARNING,
"Circular dependencies are not supported. The following modules depend on each other: "
+ chunk.modules.joinToString(", ") { it.name } + " "
+ "Kotlin is not compiled for these modules"
)
return true
}
return false
}
fun compilerArgumentsForChunk(chunk: ModuleChunk): CommonCompilerArguments =
chunk.representativeTarget().module.kotlinCompilerArguments
open fun doAfterBuild() {
}
abstract fun createCacheStorage(paths: BuildDataPaths): JpsIncrementalCache
/**
* Called for `ModuleChunk.representativeTarget`
*/
open fun updateChunkMappings(
chunk: ModuleChunk,
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
outputItems: Map<ModuleBuildTarget, Iterable<GeneratedFile>>,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
) {
// by default do nothing
}
open fun updateCaches(
jpsIncrementalCache: JpsIncrementalCache,
files: List<GeneratedFile>,
changesCollector: ChangesCollector,
environment: JpsCompilerEnvironment
) {
val expectActualTracker = environment.services[ExpectActualTracker::class.java] as ExpectActualTrackerImpl
jpsIncrementalCache.registerComplementaryFiles(expectActualTracker)
}
open fun makeServices(
builder: Services.Builder,
incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>,
lookupTracker: LookupTracker,
exceptActualTracer: ExpectActualTracker
) {
with(builder) {
register(LookupTracker::class.java, lookupTracker)
register(ExpectActualTracker::class.java, exceptActualTracer)
register(CompilationCanceledStatus::class.java, object : CompilationCanceledStatus {
override fun checkCanceled() {
if (context.cancelStatus.isCanceled) throw CompilationCanceledException()
}
})
}
}
protected fun collectSourcesToCompile(dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder) =
collectSourcesToCompile(this, dirtyFilesHolder)
/**
* Should be used only for particular target in chunk (jvm)
*/
protected fun collectSourcesToCompile(
target: KotlinModuleBuildTarget,
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder
): Collection<File> {
// Should not be cached since may be vary in different rounds
val jpsModuleTarget = target.jpsModuleBuildTarget
return if (IncrementalCompilation.isEnabled()) dirtyFilesHolder.getDirtyFiles(jpsModuleTarget)
else target.sourceFiles
}
protected fun checkShouldCompileAndLog(dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder, moduleSources: Collection<File>) =
checkShouldCompileAndLog(this, dirtyFilesHolder, moduleSources)
/**
* Should be used only for particular target in chunk (jvm)
*/
protected fun checkShouldCompileAndLog(
target: KotlinModuleBuildTarget,
dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
moduleSources: Collection<File>
): Boolean {
val hasRemovedSources = dirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget).isNotEmpty()
val hasDirtyOrRemovedSources = moduleSources.isNotEmpty() || hasRemovedSources
if (hasDirtyOrRemovedSources) {
val logger = context.loggingManager.projectBuilderLogger
if (logger.isEnabled) {
logger.logCompiledFiles(moduleSources, KotlinBuilder.KOTLIN_BUILDER_NAME, "Compiling files:")
}
}
return hasDirtyOrRemovedSources
}
open fun checkCachesVersions(chunk: ModuleChunk, dataManager: BuildDataManager, actions: MutableSet<CacheVersion.Action>) {
}
}
@@ -1,176 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.jps.platforms
import com.intellij.util.containers.MultiMap
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.builders.DirtyFilesHolder
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.ProjectBuildException
import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.model.module.JpsModuleSourceRootType
import org.jetbrains.jps.util.JpsPathUtil
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
import org.jetbrains.kotlin.config.KotlinSourceRootType
import org.jetbrains.kotlin.jps.build.FSOperationsHelper
import org.jetbrains.kotlin.jps.build.KotlinSourceFileCollector
import org.jetbrains.kotlin.jps.model.kotlinFacet
import org.jetbrains.kotlin.jps.model.productionOutputFilePath
import org.jetbrains.kotlin.jps.model.testOutputFilePath
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.utils.addIfNotNull
import java.io.File
/**
* Properties and actions for Kotlin test / production module build target.
*/
abstract class KotlinModuleBuilderTarget(val context: CompileContext, val jpsModuleBuildTarget: ModuleBuildTarget) {
val sourceRootTypes: Collection<JpsModuleSourceRootType<*>>
get() = if (isTests) {
listOf(JavaSourceRootType.TEST_SOURCE, KotlinSourceRootType.TestSource)
} else {
listOf(JavaSourceRootType.SOURCE, KotlinSourceRootType.Source)
}
val module: JpsModule
get() = jpsModuleBuildTarget.module
val isTests: Boolean
get() = jpsModuleBuildTarget.isTests
val targetId: TargetId
get() {
// Since IDEA 2016 each gradle source root is imported as a separate module.
// One gradle module X is imported as two JPS modules:
// 1. X-production with one production target;
// 2. X-test with one test target.
// This breaks kotlin code since internal members' names are mangled using module name.
// For example, a declaration of a function 'f' in 'X-production' becomes 'fXProduction', but a call 'f' in 'X-test' becomes 'fXTest()'.
// The workaround is to replace a name of such test target with the name of corresponding production module.
// See KT-11993.
val name = relatedProductionModule?.name ?: jpsModuleBuildTarget.id
return TargetId(name, jpsModuleBuildTarget.targetType.typeId)
}
val outputDir by lazy {
val explicitOutputPath = if (isTests) module.testOutputFilePath else module.productionOutputFilePath
val explicitOutputDir = explicitOutputPath?.let { File(it).absoluteFile.parentFile }
return@lazy explicitOutputDir
?: jpsModuleBuildTarget.outputDir
?: throw ProjectBuildException("No output directory found for " + this)
}
val friendBuildTargets: List<KotlinModuleBuilderTarget>
get() {
val result = mutableListOf<KotlinModuleBuilderTarget>()
if (isTests) {
result.addIfNotNull(context.kotlinBuildTargets[module.productionBuildTarget])
result.addIfNotNull(context.kotlinBuildTargets[relatedProductionModule?.productionBuildTarget])
}
return result.filter { it.sources.isNotEmpty() }
}
val friendOutputDirs: List<File>
get() = friendBuildTargets.mapNotNull {
JpsJavaExtensionService.getInstance().getOutputDirectory(it.module, false)
}
private val relatedProductionModule: JpsModule?
get() = JpsJavaExtensionService.getInstance().getTestModuleProperties(module)?.productionModule
val allDependencies by lazy {
JpsJavaExtensionService.dependencies(module).recursively().exportedOnly()
.includedIn(JpsJavaClasspathKind.compile(isTests))
}
val expectedBy by lazy(::findExpectedBy)
private fun findExpectedBy(): List<KotlinModuleBuilderTarget> {
val kotlinFacetExtension = module.kotlinFacet
val implementedModuleNames = kotlinFacetExtension?.settings?.implementedModuleNames ?: return listOf()
if (implementedModuleNames.isEmpty()) return listOf()
return allDependencies.modules
.filter { it.name in implementedModuleNames }
.map { context.kotlinBuildTargets[ModuleBuildTarget(it, isTests)]!! }
}
val sources by lazy {
mutableListOf<File>().also { result ->
// add all common libs sources
expectedBy.forEach {
it.collectSources(result)
}
collectSources(result)
}
}
private fun collectSources(receiver: MutableList<File>) {
val moduleExcludes = module.excludeRootsList.urls.mapTo(java.util.HashSet(), JpsPathUtil::urlToFile)
val compilerExcludes = JpsJavaExtensionService.getInstance()
.getOrCreateCompilerConfiguration(module.project)
.compilerExcludes
module
.sourceRoots
.filter { it.rootType in sourceRootTypes }
.forEach {
it.file.walkTopDown()
.onEnter { it !in moduleExcludes }
.filterTo(receiver) {
!compilerExcludes.isExcluded(it) &&
it.isFile &&
KotlinSourceFileCollector.isKotlinSourceFile(it)
}
}
}
override fun toString() = jpsModuleBuildTarget.toString()
abstract fun compileModuleChunk(
allCompiledFiles: MutableSet<File>,
chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
environment: JpsCompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
fsOperations: FSOperationsHelper
): Boolean
protected fun reportAndSkipCircular(
chunk: ModuleChunk,
environment: JpsCompilerEnvironment
): Boolean {
if (chunk.modules.size > 1) {
// We do not support circular dependencies, but if they are present, we do our best should not break the build,
// so we simply yield a warning and report NOTHING_DONE
environment.messageCollector.report(
CompilerMessageSeverity.STRONG_WARNING,
"Circular dependencies are not supported. The following modules depend on each other: "
+ chunk.modules.joinToString(", ") { it.name } + " "
+ "Kotlin is not compiled for these modules"
)
return true
}
return false
}
open fun doAfterBuild() {
}
}
@@ -6,7 +6,6 @@
package org.jetbrains.kotlin.jps.platforms
import com.intellij.openapi.util.Key
import org.jetbrains.jps.builders.ModuleBasedBuildTargetType
import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType
import org.jetbrains.jps.incremental.CompileContext
import org.jetbrains.jps.incremental.ModuleBuildTarget
@@ -46,17 +45,17 @@ val CompileContext.kotlinBuildTargets: KotlinBuildTargets
}
class KotlinBuildTargets internal constructor(val compileContext: CompileContext) {
private val byJpsModuleBuildTarget = ConcurrentHashMap<ModuleBuildTarget, KotlinModuleBuilderTarget>()
private val byJpsModuleBuildTarget = ConcurrentHashMap<ModuleBuildTarget, KotlinModuleBuildTarget>()
private val isKotlinJsStdlibJar = ConcurrentHashMap<String, Boolean>()
@JvmName("getNullable")
operator fun get(target: ModuleBuildTarget?): KotlinModuleBuilderTarget? {
operator fun get(target: ModuleBuildTarget?): KotlinModuleBuildTarget? {
if (target == null) return null
return get(target)
}
operator fun get(target: ModuleBuildTarget): KotlinModuleBuilderTarget? {
if (target.targetType !is ModuleBasedBuildTargetType) return null
operator fun get(target: ModuleBuildTarget): KotlinModuleBuildTarget? {
if (target.module.moduleType != JpsJavaModuleType.INSTANCE) return null
return byJpsModuleBuildTarget.computeIfAbsent(target) {
when (target.module.targetPlatform ?: detectTargetPlatform(target)) {