jps: fixes for KOTLIN-CR-2028

Original commit: c24aa13f12
2018-06-04 11:42:54 +03:00
parent 53f04d0d19
commit 304a537c68
10 changed files with 55 additions and 64 deletions
@@ -60,10 +60,6 @@ class FSOperationsHelper(
        markFilesImpl(files, beforeRound = true) { it.exists() && moduleBasedFilter.accept(it) }
    }

-    fun markComplementaryFiles(files: Iterable<File>) {
-        markFilesImpl(files, beforeRound = true) { it.exists() }
-    }
-
    fun markFiles(files: Iterable<File>) {
        markFilesImpl(files, beforeRound = false) { it.exists() }
    }
@@ -154,7 +154,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
        context: CompileContext,
        fsOperations: FSOperationsHelper
    ) {
-        val roundDirtyFiles = KotlinRoundDirtySourceFilesHolder(
+        val roundDirtyFiles = KotlinDirtySourceFilesHolder(
            chunk,
            context,
            fsOperations,
@@ -275,7 +275,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
            return NOTHING_DONE
        }

-        val roundDirtyFilesHolder = KotlinRoundDirtySourceFilesHolder(chunk, context, fsOperations, dirtyFilesHolder)
+        val roundDirtyFilesHolder = KotlinDirtySourceFilesHolder(chunk, context, fsOperations, dirtyFilesHolder)
        val projectDescriptor = context.projectDescriptor
        val dataManager = projectDescriptor.dataManager
        val targets = chunk.targets
@@ -502,7 +502,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
        kotlinTarget: KotlinModuleBuildTarget,
        commonArguments: CommonCompilerArguments,
        context: CompileContext,
-        roundDirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        environment: JpsCompilerEnvironment,
        incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
    ): OutputItemsCollector? {
@@ -529,16 +529,16 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
        if (IncrementalCompilation.isEnabled()) {
            for (target in chunk.targets) {
                val cache = incrementalCaches[target]
-                val targetDirtyFiles = roundDirtyFilesHolder.byTarget[target]
+                val targetDirtyFiles = dirtyFilesHolder.byTarget[target]

                if (cache != null && targetDirtyFiles != null) {
                    val removedAndDirtyFiles: MutableSet<File> = mutableSetOf()

-                    removedAndDirtyFiles.addAll(roundDirtyFilesHolder.getDirtyFiles(target))
-                    removedAndDirtyFiles.addAll(roundDirtyFilesHolder.getRemovedFilesSet(target))
+                    removedAndDirtyFiles.addAll(dirtyFilesHolder.getDirtyFiles(target))
+                    removedAndDirtyFiles.addAll(dirtyFilesHolder.getRemovedFilesSet(target))

                    val complementaryFiles = cache.clearComplementaryFilesMapping(targetDirtyFiles.dirtyOrRemovedFiles)
-                    targetDirtyFiles.addComplementaryFiles(complementaryFiles)
+                    targetDirtyFiles.markDirtyForCurrentRound(complementaryFiles)

                    cache.markDirty(targetDirtyFiles.dirtyOrRemovedFiles)
                }
@@ -546,7 +546,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
        }

        val isDoneSomething = kotlinTarget.compileModuleChunk(
-            chunk, commonArguments, roundDirtyFilesHolder, environment
+            chunk, commonArguments, dirtyFilesHolder, environment
        )

        return if (isDoneSomething) environment.outputItemsCollector else null
@@ -642,7 +642,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
    private fun updateLookupStorage(
        lookupTracker: LookupTracker,
        dataManager: BuildDataManager,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder
    ) {
        if (lookupTracker !is LookupTrackerImpl)
            throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
@@ -32,7 +32,7 @@ import java.io.File
 *
 * Probably should be merged with [FSOperationsHelper]
 */
-class KotlinRoundDirtySourceFilesHolder(
+class KotlinDirtySourceFilesHolder(
    val chunk: ModuleChunk,
    val context: CompileContext,
    val fsOperations: FSOperationsHelper,
@@ -51,8 +51,8 @@ class KotlinRoundDirtySourceFilesHolder(
                return result
            }

-        fun addComplementaryFiles(files: Collection<File>) {
-            fsOperations.markComplementaryFiles(files)
+        fun markDirtyForCurrentRound(files: Collection<File>) {
+            fsOperations.markFilesBeforeInitialRound(files)
            files.forEach {
                dirty.add(DirtyFile(it, null))
            }
@@ -11,17 +11,24 @@ import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType
 import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
 import org.jetbrains.jps.builders.storage.BuildDataPaths
 import org.jetbrains.jps.incremental.ModuleBuildTarget
+import org.jetbrains.jps.model.java.JavaResourceRootType
 import org.jetbrains.jps.model.java.JavaSourceRootProperties
 import org.jetbrains.jps.model.java.JavaSourceRootType
 import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.model.module.JpsModuleSourceRootType
+import org.jetbrains.kotlin.config.KotlinResourceRootType
 import org.jetbrains.kotlin.config.KotlinSourceRootType
 import org.jetbrains.kotlin.jps.model.expectedByModules
 import java.io.File

 /**
- * - adds roots with KotlinSourceRootType
+ * - adds roots with KotlinSourceRootType as JavaSourceRootDescriptor (see note below)
 * - for Multiplatform Projects: adds all the source roots of the expectedBy modules to the platform modules.
+ *
+ * Note: `KotlinSourceRootType` cannot be supported directly, since `SourceRootDescriptors` are computed by
+ * `ModuleBuildTarget.computeAllTargets`. `ModuleBuildTarget` is required for incremental compilation.
+ * We cannot define our own `ModuleBuildTarget` since it is final and `ModuleBuildTarget` supports only `JavaSourceRootDescriptor`.
+ * So the only one way to support `KotlinSourceRootType` is to add a fake `JavaSourceRootDescriptor` for each source root with that type.
 */
 class KotlinSourceRootProvider : AdditionalRootsProviderService<JavaSourceRootDescriptor>(JavaModuleBuildTargetType.ALL_TYPES) {
    override fun getAdditionalRoots(
@@ -63,12 +70,11 @@ class KotlinSourceRootProvider : AdditionalRootsProviderService<JavaSourceRootDe
    ) {
        for (commonSourceRoot in commonModule.sourceRoots) {
            val isCommonTestsRootType = commonSourceRoot.rootType.isTestsRootType
-            if (isCommonTestsRootType == null || target.isTests == isCommonTestsRootType) {
+            if (isCommonTestsRootType != null && target.isTests == isCommonTestsRootType) {
                val javaSourceRootProperties = commonSourceRoot.properties as? JavaSourceRootProperties

                result.add(
                    KotlinCommonModuleSourceRoot(
-                        commonModule,
                        commonSourceRoot.file,
                        target,
                        javaSourceRootProperties?.isForGeneratedSources ?: false,
@@ -85,13 +91,14 @@ class KotlinSourceRootProvider : AdditionalRootsProviderService<JavaSourceRootDe
 private val JpsModuleSourceRootType<*>.isTestsRootType
    get() = when (this) {
        is KotlinSourceRootType -> this == KotlinSourceRootType.TestSource
-        is JavaSourceRootType -> this == JavaSourceRootType.TEST_SOURCE // for compatibility
+        is KotlinResourceRootType -> this == KotlinResourceRootType.TestResource
+    // for compatibility:
+        is JavaSourceRootType -> this == JavaSourceRootType.TEST_SOURCE
+        is JavaResourceRootType -> this == JavaResourceRootType.TEST_RESOURCE
        else -> null
    }

-
 class KotlinCommonModuleSourceRoot(
-    val commonModule: JpsModule,
    root: File,
    target: ModuleBuildTarget,
    isGenerated: Boolean,
@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.config.CompilerRunnerConstants
 import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget
+import java.io.File

 class MessageCollectorAdapter(
    private val context: CompileContext,
@@ -31,7 +32,7 @@ class MessageCollectorAdapter(
        val kind = kind(severity)
        if (kind != null) {
            // Report target when cross-compiling common files
-            if (location != null && kotlinTarget != null && kotlinTarget.isCommonModuleFile(location.path)) {
+            if (location != null && kotlinTarget != null && kotlinTarget.isCommonModuleFile(File(location.path))) {
                val moduleName = kotlinTarget.module.name
                prefix += "[$moduleName] "
            }
@@ -37,7 +37,9 @@ class JpsIncrementalJvmCache(
    paths: BuildDataPaths
 ) : IncrementalJvmCache(paths.getTargetDataRoot(target), target.outputDir), JpsIncrementalCache {
    override fun addJpsDependentCache(cache: JpsIncrementalCache) {
-        addDependentCache(cache as JpsIncrementalJvmCache)
+        if (cache is JpsIncrementalJvmCache) {
+            addDependentCache(cache)
+        }
    }

    override fun debugLog(message: String) {
@@ -5,7 +5,6 @@

 package org.jetbrains.kotlin.jps.platforms

-import com.intellij.util.xmlb.XmlSerializerUtil
 import org.jetbrains.jps.ModuleChunk
 import org.jetbrains.jps.builders.storage.BuildDataPaths
 import org.jetbrains.jps.incremental.CompileContext
@@ -15,10 +14,9 @@ import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.util.JpsPathUtil
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
-import org.jetbrains.kotlin.cli.common.arguments.mergeBeans
 import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment
 import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner
-import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
+import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
 import org.jetbrains.kotlin.jps.model.k2MetadataCompilerArguments
 import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings
@@ -29,7 +27,7 @@ class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarge
    override fun compileModuleChunk(
        chunk: ModuleChunk,
        commonArguments: CommonCompilerArguments,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        environment: JpsCompilerEnvironment
    ): Boolean {
        reportAndSkipCircular(chunk, environment)
@@ -81,7 +79,7 @@ class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarge

        if (dependencyBuildTarget != this@KotlinCommonModuleBuildTarget &&
            dependencyBuildTarget is KotlinCommonModuleBuildTarget &&
-            dependencyBuildTarget.sourceFiles.isNotEmpty()
+            dependencyBuildTarget.sources.isNotEmpty()
        ) {
            result.add(dependencyBuildTarget.destination)
        }
@@ -22,11 +22,8 @@ import org.jetbrains.kotlin.incremental.ChangesCollector
 import org.jetbrains.kotlin.incremental.IncrementalJsCache
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
-import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
-import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
-import org.jetbrains.kotlin.incremental.js.TranslationResultValue
-import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
+import org.jetbrains.kotlin.incremental.js.*
+import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJsCache
 import org.jetbrains.kotlin.jps.model.k2JsCompilerArguments
@@ -39,14 +36,6 @@ import org.jetbrains.kotlin.utils.KotlinJavascriptMetadataUtils.META_JS_SUFFIX
 import java.io.File
 import java.net.URI

-internal class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : IncrementalDataProvider {
-    override val headerMetadata: ByteArray
-        get() = cache.header
-
-    override val compiledPackageParts: Map<File, TranslationResultValue>
-        get() = cache.nonDirtyPackageParts()
-}
-
 class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) :
    KotlinModuleBuildTarget(compileContext, jpsModuleBuildTarget) {

@@ -81,7 +70,7 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa
    override fun compileModuleChunk(
        chunk: ModuleChunk,
        commonArguments: CommonCompilerArguments,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        environment: JpsCompilerEnvironment
    ): Boolean {
        require(chunk.representativeTarget() == jpsModuleBuildTarget)
@@ -188,7 +177,7 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa

        if (dependencyBuildTarget != this@KotlinJsModuleBuildTarget &&
            dependencyBuildTarget is KotlinJsModuleBuildTarget &&
-            dependencyBuildTarget.sourceFiles.isNotEmpty()
+            dependencyBuildTarget.sources.isNotEmpty()
        ) {
            val metaFile = dependencyBuildTarget.outputMetaFile
            if (metaFile.exists()) {
@@ -34,7 +34,7 @@ import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.jps.build.KotlinBuilder
-import org.jetbrains.kotlin.jps.build.KotlinRoundDirtySourceFilesHolder
+import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
 import org.jetbrains.kotlin.jps.build.jvmBuildMetaInfoFile
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
 import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache
@@ -121,7 +121,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
    override fun compileModuleChunk(
        chunk: ModuleChunk,
        commonArguments: CommonCompilerArguments,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        environment: JpsCompilerEnvironment
    ): Boolean {
        if (chunk.modules.size > 1) {
@@ -174,7 +174,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT
        return true
    }

-    fun generateModuleDescription(chunk: ModuleChunk, dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder): File? {
+    fun generateModuleDescription(chunk: ModuleChunk, dirtyFilesHolder: KotlinDirtySourceFilesHolder): File? {
        val builder = KotlinModuleXmlBuilder()

        var hasDirtySources = false
@@ -294,7 +294,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT

    override fun updateChunkMappings(
        chunk: ModuleChunk,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        outputItems: Map<ModuleBuildTarget, Iterable<GeneratedFile>>,
        incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
    ) {
@@ -11,7 +11,6 @@ import org.jetbrains.jps.incremental.CompileContext
 import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jps.incremental.ProjectBuildException
 import org.jetbrains.jps.incremental.storage.BuildDataManager
-import org.jetbrains.jps.model.java.JavaSourceRootType
 import org.jetbrains.jps.model.java.JpsJavaClasspathKind
 import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import org.jetbrains.jps.model.module.JpsModule
@@ -79,7 +78,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
                result.addIfNotNull(context.kotlinBuildTargets[relatedProductionModule?.productionBuildTarget])
            }

-            return result.filter { it.sourceFiles.isNotEmpty() }
+            return result.filter { it.sources.isNotEmpty() }
        }

    val friendOutputDirs: List<File>
@@ -95,13 +94,13 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
            .includedIn(JpsJavaClasspathKind.compile(isTests))
    }

-    val sources by lazy {
-        mutableMapOf<String, Source>().also { result ->
+    val sources: Map<File, Source> by lazy {
+        mutableMapOf<File, Source>().also { result ->
            collectSources(result)
        }
    }

-    private fun collectSources(receiver: MutableMap<String, Source>) {
+    private fun collectSources(receiver: MutableMap<File, Source>) {
        val moduleExcludes = module.excludeRootsList.urls.mapTo(java.util.HashSet(), JpsPathUtil::urlToFile)

        val compilerExcludes = JpsJavaExtensionService.getInstance()
@@ -117,7 +116,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
                .onEnter { file -> file !in moduleExcludes }
                .forEach { file ->
                    if (!compilerExcludes.isExcluded(file) && file.isFile && file.isKotlinSourceFile) {
-                        receiver[file.path] = Source(file, isCommonRoot)
+                        receiver[file] = Source(file, isCommonRoot)
                    }
                }

@@ -132,11 +131,10 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
        val isCommonModule: Boolean
    )

-    fun isCommonModuleFile(path: String): Boolean = sources[path]?.isCommonModule == true
+    fun isCommonModuleFile(file: File): Boolean = sources[file]?.isCommonModule == true

-    val sourceFiles by lazy {
-        sources.values.map { it.file }
-    }
+    val sourceFiles: Collection<File>
+        get() = sources.values.map { it.file }

    override fun toString() = jpsModuleBuildTarget.toString()

@@ -146,7 +144,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
    abstract fun compileModuleChunk(
        chunk: ModuleChunk,
        commonArguments: CommonCompilerArguments,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        environment: JpsCompilerEnvironment
    ): Boolean

@@ -183,7 +181,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
     */
    open fun updateChunkMappings(
        chunk: ModuleChunk,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        outputItems: Map<ModuleBuildTarget, Iterable<GeneratedFile>>,
        incrementalCaches: Map<ModuleBuildTarget, JpsIncrementalCache>
    ) {
@@ -217,7 +215,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
        }
    }

-    protected fun collectSourcesToCompile(dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder) =
+    protected fun collectSourcesToCompile(dirtyFilesHolder: KotlinDirtySourceFilesHolder) =
        collectSourcesToCompile(this, dirtyFilesHolder)

    /**
@@ -225,7 +223,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
     */
    protected fun collectSourcesToCompile(
        target: KotlinModuleBuildTarget,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder
    ): Collection<File> {
        // Should not be cached since may be vary in different rounds

@@ -234,7 +232,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
        else target.sourceFiles
    }

-    protected fun checkShouldCompileAndLog(dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder, moduleSources: Collection<File>) =
+    protected fun checkShouldCompileAndLog(dirtyFilesHolder: KotlinDirtySourceFilesHolder, moduleSources: Collection<File>) =
        checkShouldCompileAndLog(this, dirtyFilesHolder, moduleSources)

    /**
@@ -242,7 +240,7 @@ abstract class KotlinModuleBuildTarget(val context: CompileContext, val jpsModul
     */
    protected fun checkShouldCompileAndLog(
        target: KotlinModuleBuildTarget,
-        dirtyFilesHolder: KotlinRoundDirtySourceFilesHolder,
+        dirtyFilesHolder: KotlinDirtySourceFilesHolder,
        moduleSources: Collection<File>
    ): Boolean {
        val hasRemovedSources = dirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget).isNotEmpty()