[JPS] Fix JS incremental compilation

Disable Idea JPS build mechanism for marking all sources of common output if one of them is dirty
Add source-to-outputs map for correctly removing Kotlin/JS outputs

#KT-45763 Fixed
#KT-44351 Fixed

build-common/src/org/jetbrains/kotlin/build/generatedFiles.kt

@@ -39,4 +39,4 @@ class GeneratedJvmClass(
     }
 }
 
-fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF"
+fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF"

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.incremental
 
 import com.intellij.util.io.DataExternalizer
+import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
 import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
 import org.jetbrains.kotlin.incremental.js.TranslationResultValue
@@ -48,6 +49,7 @@ open class IncrementalJsCache(
         private const val INLINE_FUNCTIONS = "inline-functions"
         private const val HEADER_FILE_NAME = "header.meta"
         private const val PACKAGE_META_FILE = "packages-meta"
+        private const val SOURCE_TO_JS_OUTPUT = "source-to-js-output"
 
         fun hasHeaderFile(cachesDir: File) = File(cachesDir, HEADER_FILE_NAME).exists()
     }
@@ -60,6 +62,7 @@ open class IncrementalJsCache(
     private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter))
     private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter))
     private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile))
+    private val sourceToJsOutputsMap = registerMap(SourceToJsOutputMap(SOURCE_TO_JS_OUTPUT.storageFile, pathConverter))
 
     private val dirtySources = hashSetOf<File>()
 
@@ -75,6 +78,7 @@ open class IncrementalJsCache(
 
     override fun markDirty(removedAndCompiledSources: Collection<File>) {
         removedAndCompiledSources.forEach { sourceFile ->
+            sourceToJsOutputsMap.remove(sourceFile)
             // The common prefix of all FQN parents has to be the file package
             sourceToClassesMap[sourceFile].map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let {
                 packageMetadata.remove(it)
@@ -95,6 +99,10 @@ open class IncrementalJsCache(
         }
     }
 
+    fun getOutputsBySource(sourceFile: File): Collection<File> {
+        return sourceToJsOutputsMap.get(sourceFile)
+    }
+
     fun compareAndUpdate(incrementalResults: IncrementalResultsConsumerImpl, changesCollector: ChangesCollector) {
         val translatedFiles = incrementalResults.packageParts
 
@@ -175,6 +183,17 @@ open class IncrementalJsCache(
                 }
             }
         }
+
+    fun updateSourceToOutputMap(
+        generatedFiles: Iterable<GeneratedFile>,
+    ) {
+        for (generatedFile in generatedFiles) {
+            for (source in generatedFile.sourceFiles) {
+                if (dirtySources.contains(source))
+                    sourceToJsOutputsMap.add(source, generatedFile.outputFile)
+            }
+        }
+    }
 }
 
 private object TranslationResultValueExternalizer : DataExternalizer<TranslationResultValue> {

build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.storage
+
+import org.jetbrains.kotlin.incremental.dumpCollection
+import java.io.File
+
+class SourceToJsOutputMap(storageFile: File, private val pathConverter: FileToPathConverter) : BasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer) {
+    override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
+
+    @Synchronized
+    fun add(key: File, value: File) {
+        storage.append(pathConverter.toPath(key), listOf(pathConverter.toPath(value)))
+    }
+
+    operator fun get(sourceFile: File): Collection<File> =
+        storage[pathConverter.toPath(sourceFile)]?.map { pathConverter.toFile(it) } ?: setOf()
+
+
+    @Synchronized
+    operator fun set(key: File, values: Collection<File>) {
+        if (values.isEmpty()) {
+            remove(key)
+            return
+        }
+
+        storage[pathConverter.toPath(key)] = values.map { pathConverter.toPath(it) }
+    }
+
+    @Synchronized
+    fun remove(key: File) {
+        storage.remove(pathConverter.toPath(key))
+    }
+
+    @Synchronized
+    fun removeValues(key: File, removed: Set<File>) {
+        val notRemoved = this[key].filter { it !in removed }
+        this[key] = notRemoved
+    }
+}

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt

@@ -25,6 +25,7 @@ import org.jetbrains.jps.builders.java.JavaBuilderUtil
 import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
 import org.jetbrains.jps.builders.storage.BuildDataCorruptedException
 import org.jetbrains.jps.incremental.*
+import org.jetbrains.jps.incremental.BuildOperations.deleteRecursively
 import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.*
 import org.jetbrains.jps.incremental.java.JavaBuilder
 import org.jetbrains.jps.model.JpsProject
@@ -32,8 +33,8 @@ import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.GeneratedJvmClass
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.INFO
 import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil
 import org.jetbrains.kotlin.compilerRunner.*
 import org.jetbrains.kotlin.config.IncrementalCompilation
@@ -291,7 +292,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
         if (chunk.modules.any { it.kotlinKind == KotlinModuleKind.SOURCE_SET_HOLDER }) {
             if (chunk.modules.size > 1) {
                 messageCollector.report(
-                    CompilerMessageSeverity.ERROR,
+                    ERROR,
                     "Cyclically dependent modules are not supported in multiplatform projects"
                 )
                 return ABORT
@@ -410,6 +411,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
             kotlinDirtyFilesHolder.allRemovedFilesFiles
         )
 
+        cleanJsOutputs(context, kotlinChunk, incrementalCaches, kotlinDirtyFilesHolder)
+
         if (LOG.isDebugEnabled) {
             LOG.debug("Compiling files: ${kotlinDirtyFilesHolder.allDirtyFiles}")
         }
@@ -509,6 +512,43 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
         return OK
     }
 
+    private fun cleanJsOutputs(
+        context: CompileContext,
+        kotlinChunk: KotlinChunk,
+        incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
+        kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder
+    ) {
+        for (target in kotlinChunk.targets) {
+            val cache = incrementalCaches[target] ?: continue
+
+            if (cache is IncrementalJsCache) {
+                val filesToDelete = mutableListOf<File>()
+                val dirtyFiles = kotlinDirtyFilesHolder.getDirtyFiles(target.jpsModuleBuildTarget).keys
+                val removedFiles = kotlinDirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget)
+
+                for (file: File in dirtyFiles + removedFiles) {
+                    filesToDelete.addAll(cache.getOutputsBySource(file).filter { it !in filesToDelete })
+                }
+
+                if (filesToDelete.isNotEmpty()) {
+                    val deletedForThisSource = mutableSetOf<String>()
+                    val parentDirs = mutableSetOf<File>()
+
+                    for (kjsmFile in filesToDelete) {
+                        deleteRecursively(kjsmFile.path, deletedForThisSource, parentDirs)
+                    }
+
+                    FSOperations.pruneEmptyDirs(context, parentDirs)
+
+                    val logger = context.loggingManager.projectBuilderLogger
+                    if (logger.isEnabled && deletedForThisSource.isNotEmpty()) {
+                        logger.logDeletedFiles(deletedForThisSource)
+                    }
+                }
+            }
+        }
+    }
+
     // todo(1.2.80): got rid of ModuleChunk (replace with KotlinChunk)
     // todo(1.2.80): introduce KotlinRoundCompileContext, move dirtyFilesHolder, fsOperations, environment to it
     private fun doCompileModuleChunk(

jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.jps.targets
 
 import org.jetbrains.jps.builders.storage.BuildDataPaths
 import org.jetbrains.jps.incremental.ModuleBuildTarget
+import org.jetbrains.jps.incremental.ModuleLevelBuilder
 import org.jetbrains.jps.model.library.JpsOrderRootType
 import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.util.JpsPathUtil
@@ -224,7 +225,20 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
         val jsCache = jpsIncrementalCache as IncrementalJsCache
         jsCache.header = incrementalResults.headerMetadata
 
+        jsCache.updateSourceToOutputMap(files)
         jsCache.compareAndUpdate(incrementalResults, changesCollector)
         jsCache.clearCacheForRemovedClasses(changesCollector)
     }
+
+    override fun registerOutputItems(outputConsumer: ModuleLevelBuilder.OutputConsumer, outputItems: List<GeneratedFile>) {
+        if (isIncrementalCompilationEnabled) {
+            for (output in outputItems) {
+                for (source in output.sourceFiles) {
+                    outputConsumer.registerOutputFile(jpsModuleBuildTarget, File("${source.path.hashCode()}"), listOf(source.path))
+                }
+            }
+        } else {
+            super.registerOutputItems(outputConsumer, outputItems)
+        }
+    }
 }

jps-plugin/testData/incremental/multiModule/common/classRemoved/js-jps-build.log

@@ -3,11 +3,6 @@
 Building module1
 Exit code: NOTHING_DONE
 ------------------------------------------
-Cleaning output files:
-  out/production/module2/module2.js
-  out/production/module2/module2.meta.js
-  out/production/module2/module2/foo/foo.kjsm
-End of files
 Building module2
 Marked as dirty by Kotlin:
   module2/src/B.kt
@@ -16,6 +11,9 @@ Marked as dirty by Kotlin:
   module2/src/useAfoo.kt
   module2/src/useBbar.kt
 Cleaning output files:
+  out/production/module2/module2.js
+  out/production/module2/module2.meta.js
+  out/production/module2/module2/foo/foo.kjsm
   out/production/module2/module2/use/use.kjsm
 End of files
 Compiling files:

jps-plugin/testData/incremental/multiModule/common/moveFileToAnotherModule/js-jps-build.log

@@ -11,15 +11,13 @@ Exit code: ADDITIONAL_PASS_REQUIRED
 ------------------------------------------
 Exit code: NOTHING_DONE
 ------------------------------------------
-Cleaning output files:
-  out/production/module2/module2.js
-  out/production/module2/module2.meta.js
-  out/production/module2/module2/b/b.kjsm
-End of files
 Building module2
 Marked as dirty by Kotlin:
   module2/src/useClassB.kt
 Cleaning output files:
+  out/production/module2/module2.js
+  out/production/module2/module2.meta.js
+  out/production/module2/module2/b/b.kjsm
   out/production/module2/module2/usage/usage.kjsm
 End of files
 Compiling files:

jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/ignoreAndWarnAboutNative/editingCKotlin/build.log

@@ -63,12 +63,12 @@ Exit code: NOTHING_DONE
 Building pNative1
 Exit code: NOTHING_DONE
 ------------------------------------------
+Building pJs
 Cleaning output files:
   out/production/pJs/pJs.js
   out/production/pJs/pJs.meta.js
   out/production/pJs/pJs/root-package.kjsm
 End of files
-Building pJs
 Compiling files:
 End of files
 Exit code: OK

jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simple/editingCKotlin/build.log

@@ -48,12 +48,12 @@ Exit code: OK
 Building c
 Exit code: NOTHING_DONE
 ------------------------------------------
+Building pJs
 Cleaning output files:
   out/production/pJs/pJs.js
   out/production/pJs/pJs.meta.js
   out/production/pJs/pJs/root-package.kjsm
 End of files
-Building pJs
 Compiling files:
 End of files
 Exit code: OK

jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simple/editingPJsKotlin/build.log

@@ -38,12 +38,12 @@ Exit code: NOTHING_DONE
 Building c
 Exit code: NOTHING_DONE
 ------------------------------------------
+Building pJs
 Cleaning output files:
   out/production/pJs/pJs.js
   out/production/pJs/pJs.meta.js
   out/production/pJs/pJs/root-package.kjsm
 End of files
-Building pJs
 Compiling files:
 End of files
 Exit code: OK

jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simpleNewMpp/editingCKotlin/build.log

@@ -42,12 +42,12 @@ Exit code: OK
 ================ Step #3 delete new service =================
 
 Building c
+Building pJs
 Cleaning output files:
   out/production/pJs/pJs.js
   out/production/pJs/pJs.meta.js
   out/production/pJs/pJs/root-package.kjsm
 End of files
-Building pJs
 Compiling files:
 End of files
 Exit code: OK

jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simpleNewMpp/editingPJsKotlin/build.log

@@ -32,12 +32,12 @@ Exit code: NOTHING_DONE
 ================ Step #3 delete new service =================
 
 Building c
+Building pJs
 Cleaning output files:
   out/production/pJs/pJs.js
   out/production/pJs/pJs.meta.js
   out/production/pJs/pJs/root-package.kjsm
 End of files
-Building pJs
 Compiling files:
 End of files
 Exit code: OK