azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JPS] Fix JS incremental compilation

Browse Source 
Disable Idea JPS build mechanism for marking all sources of common output if one of them is dirty
Add source-to-outputs map for correctly removing Kotlin/JS outputs

#KT-45763 Fixed
#KT-44351 Fixed
This commit is contained in:
Aleksei.Cherepanov
2021-05-21 12:35:05 +03:00
committed by TeamCityServer
parent c2389a94fa
commit 5f4be07225
12 changed files with 131 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/build/generatedFiles.kt
+1 -1
View File
@@ -39,4 +39,4 @@ class GeneratedJvmClass(
}
}
fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF"
fun File.isModuleMappingFile() = extension == ModuleMapping.MAPPING_FILE_EXT && parentFile.name == "META-INF"
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt
+19
View File
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.incremental
import com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
import org.jetbrains.kotlin.incremental.js.TranslationResultValue
@@ -48,6 +49,7 @@ open class IncrementalJsCache(
private const val INLINE_FUNCTIONS = "inline-functions"
private const val HEADER_FILE_NAME = "header.meta"
private const val PACKAGE_META_FILE = "packages-meta"
private const val SOURCE_TO_JS_OUTPUT = "source-to-js-output"
fun hasHeaderFile(cachesDir: File) = File(cachesDir, HEADER_FILE_NAME).exists()
}
@@ -60,6 +62,7 @@ open class IncrementalJsCache(
private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter))
private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter))
private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile))
private val sourceToJsOutputsMap = registerMap(SourceToJsOutputMap(SOURCE_TO_JS_OUTPUT.storageFile, pathConverter))
private val dirtySources = hashSetOf<File>()
@@ -75,6 +78,7 @@ open class IncrementalJsCache(
override fun markDirty(removedAndCompiledSources: Collection<File>) {
removedAndCompiledSources.forEach { sourceFile ->
sourceToJsOutputsMap.remove(sourceFile)
// The common prefix of all FQN parents has to be the file package
sourceToClassesMap[sourceFile].map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let {
packageMetadata.remove(it)
@@ -95,6 +99,10 @@ open class IncrementalJsCache(
}
}
fun getOutputsBySource(sourceFile: File): Collection<File> {
return sourceToJsOutputsMap.get(sourceFile)
}
fun compareAndUpdate(incrementalResults: IncrementalResultsConsumerImpl, changesCollector: ChangesCollector) {
val translatedFiles = incrementalResults.packageParts
@@ -175,6 +183,17 @@ open class IncrementalJsCache(
}
}
}
fun updateSourceToOutputMap(
generatedFiles: Iterable<GeneratedFile>,
) {
for (generatedFile in generatedFiles) {
for (source in generatedFile.sourceFiles) {
if (dirtySources.contains(source))
sourceToJsOutputsMap.add(source, generatedFile.outputFile)
}
}
}
}
private object TranslationResultValueExternalizer : DataExternalizer<TranslationResultValue> {
build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File
class SourceToJsOutputMap(storageFile: File, private val pathConverter: FileToPathConverter) : BasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer) {
override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
@Synchronized
fun add(key: File, value: File) {
storage.append(pathConverter.toPath(key), listOf(pathConverter.toPath(value)))
}
operator fun get(sourceFile: File): Collection<File> =
storage[pathConverter.toPath(sourceFile)]?.map { pathConverter.toFile(it) } ?: setOf()
@Synchronized
operator fun set(key: File, values: Collection<File>) {
if (values.isEmpty()) {
remove(key)
return
}
storage[pathConverter.toPath(key)] = values.map { pathConverter.toPath(it) }
}
@Synchronized
fun remove(key: File) {
storage.remove(pathConverter.toPath(key))
}
@Synchronized
fun removeValues(key: File, removed: Set<File>) {
val notRemoved = this[key].filter { it !in removed }
this[key] = notRemoved
}
}
jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+43 -3
View File
@@ -25,6 +25,7 @@ import org.jetbrains.jps.builders.java.JavaBuilderUtil
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor
import org.jetbrains.jps.builders.storage.BuildDataCorruptedException
import org.jetbrains.jps.incremental.*
import org.jetbrains.jps.incremental.BuildOperations.deleteRecursively
import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.*
import org.jetbrains.jps.incremental.java.JavaBuilder
import org.jetbrains.jps.model.JpsProject
@@ -32,8 +33,8 @@ import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.INFO
import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil
import org.jetbrains.kotlin.compilerRunner.*
import org.jetbrains.kotlin.config.IncrementalCompilation
@@ -291,7 +292,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
if (chunk.modules.any { it.kotlinKind == KotlinModuleKind.SOURCE_SET_HOLDER }) {
if (chunk.modules.size > 1) {
messageCollector.report(
CompilerMessageSeverity.ERROR,
ERROR,
"Cyclically dependent modules are not supported in multiplatform projects"
)
return ABORT
@@ -410,6 +411,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
kotlinDirtyFilesHolder.allRemovedFilesFiles
)
cleanJsOutputs(context, kotlinChunk, incrementalCaches, kotlinDirtyFilesHolder)
if (LOG.isDebugEnabled) {
LOG.debug("Compiling files: ${kotlinDirtyFilesHolder.allDirtyFiles}")
}
@@ -509,6 +512,43 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
return OK
}
private fun cleanJsOutputs(
context: CompileContext,
kotlinChunk: KotlinChunk,
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder
) {
for (target in kotlinChunk.targets) {
val cache = incrementalCaches[target] ?: continue
if (cache is IncrementalJsCache) {
val filesToDelete = mutableListOf<File>()
val dirtyFiles = kotlinDirtyFilesHolder.getDirtyFiles(target.jpsModuleBuildTarget).keys
val removedFiles = kotlinDirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget)
for (file: File in dirtyFiles + removedFiles) {
filesToDelete.addAll(cache.getOutputsBySource(file).filter { it !in filesToDelete })
}
if (filesToDelete.isNotEmpty()) {
val deletedForThisSource = mutableSetOf<String>()
val parentDirs = mutableSetOf<File>()
for (kjsmFile in filesToDelete) {
deleteRecursively(kjsmFile.path, deletedForThisSource, parentDirs)
}
FSOperations.pruneEmptyDirs(context, parentDirs)
val logger = context.loggingManager.projectBuilderLogger
if (logger.isEnabled && deletedForThisSource.isNotEmpty()) {
logger.logDeletedFiles(deletedForThisSource)
}
}
}
}
}
// todo(1.2.80): got rid of ModuleChunk (replace with KotlinChunk)
// todo(1.2.80): introduce KotlinRoundCompileContext, move dirtyFilesHolder, fsOperations, environment to it
private fun doCompileModuleChunk(
jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt
+14
View File
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.jps.targets
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jps.incremental.ModuleLevelBuilder
import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
@@ -224,7 +225,20 @@ class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBu
val jsCache = jpsIncrementalCache as IncrementalJsCache
jsCache.header = incrementalResults.headerMetadata
jsCache.updateSourceToOutputMap(files)
jsCache.compareAndUpdate(incrementalResults, changesCollector)
jsCache.clearCacheForRemovedClasses(changesCollector)
}
override fun registerOutputItems(outputConsumer: ModuleLevelBuilder.OutputConsumer, outputItems: List<GeneratedFile>) {
if (isIncrementalCompilationEnabled) {
for (output in outputItems) {
for (source in output.sourceFiles) {
outputConsumer.registerOutputFile(jpsModuleBuildTarget, File("${source.path.hashCode()}"), listOf(source.path))
}
}
} else {
super.registerOutputItems(outputConsumer, outputItems)
}
}
}
jps-plugin/testData/incremental/multiModule/common/classRemoved/js-jps-build.log
Vendored
+3 -5
View File
@@ -3,11 +3,6 @@
Building module1
Exit code: NOTHING_DONE
------------------------------------------
Cleaning output files:
out/production/module2/module2.js
out/production/module2/module2.meta.js
out/production/module2/module2/foo/foo.kjsm
End of files
Building module2
Marked as dirty by Kotlin:
module2/src/B.kt
@@ -16,6 +11,9 @@ Marked as dirty by Kotlin:
module2/src/useAfoo.kt
module2/src/useBbar.kt
Cleaning output files:
out/production/module2/module2.js
out/production/module2/module2.meta.js
out/production/module2/module2/foo/foo.kjsm
out/production/module2/module2/use/use.kjsm
End of files
Compiling files:
jps-plugin/testData/incremental/multiModule/common/moveFileToAnotherModule/js-jps-build.log
Vendored
+3 -5
View File
@@ -11,15 +11,13 @@ Exit code: ADDITIONAL_PASS_REQUIRED
------------------------------------------
Exit code: NOTHING_DONE
------------------------------------------
Cleaning output files:
out/production/module2/module2.js
out/production/module2/module2.meta.js
out/production/module2/module2/b/b.kjsm
End of files
Building module2
Marked as dirty by Kotlin:
module2/src/useClassB.kt
Cleaning output files:
out/production/module2/module2.js
out/production/module2/module2.meta.js
out/production/module2/module2/b/b.kjsm
out/production/module2/module2/usage/usage.kjsm
End of files
Compiling files:
jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/ignoreAndWarnAboutNative/editingCKotlin/build.log
Vendored
+1 -1
View File
@@ -63,12 +63,12 @@ Exit code: NOTHING_DONE
Building pNative1
Exit code: NOTHING_DONE
------------------------------------------
Building pJs
Cleaning output files:
out/production/pJs/pJs.js
out/production/pJs/pJs.meta.js
out/production/pJs/pJs/root-package.kjsm
End of files
Building pJs
Compiling files:
End of files
Exit code: OK
jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simple/editingCKotlin/build.log
Vendored
+1 -1
View File
@@ -48,12 +48,12 @@ Exit code: OK
Building c
Exit code: NOTHING_DONE
------------------------------------------
Building pJs
Cleaning output files:
out/production/pJs/pJs.js
out/production/pJs/pJs.meta.js
out/production/pJs/pJs/root-package.kjsm
End of files
Building pJs
Compiling files:
End of files
Exit code: OK
jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simple/editingPJsKotlin/build.log
Vendored
+1 -1
View File
@@ -38,12 +38,12 @@ Exit code: NOTHING_DONE
Building c
Exit code: NOTHING_DONE
------------------------------------------
Building pJs
Cleaning output files:
out/production/pJs/pJs.js
out/production/pJs/pJs.meta.js
out/production/pJs/pJs/root-package.kjsm
End of files
Building pJs
Compiling files:
End of files
Exit code: OK
jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simpleNewMpp/editingCKotlin/build.log
Vendored
+1 -1
View File
@@ -42,12 +42,12 @@ Exit code: OK
================ Step #3 delete new service =================
Building c
Building pJs
Cleaning output files:
out/production/pJs/pJs.js
out/production/pJs/pJs.meta.js
out/production/pJs/pJs/root-package.kjsm
End of files
Building pJs
Compiling files:
End of files
Exit code: OK
jps-plugin/testData/incremental/multiModule/multiplatform/withGeneratedContent/simpleNewMpp/editingPJsKotlin/build.log
Vendored
+1 -1
View File
@@ -32,12 +32,12 @@ Exit code: NOTHING_DONE
================ Step #3 delete new service =================
Building c
Building pJs
Cleaning output files:
out/production/pJs/pJs.js
out/production/pJs/pJs.meta.js
out/production/pJs/pJs/root-package.kjsm
End of files
Building pJs
Compiling files:
End of files
Exit code: OK