azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jps: support multiplatform incremental compilation for jvm and js

Browse Source 
- support common modules metadata compilation under flag (it is not required since all common source roots are included transitively for now)
- introduce expect actual tracker in jps: move implementation from gradle to build-common
- support js incremental compilation: move implementation from gradle to build-common
This commit is contained in:
Sergey Rostov
2018-04-03 13:00:06 +03:00
parent 4a6554a36d
commit 0eee2729cd
89 changed files with 5829 additions and 870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheCommon.kt → build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt
+59 -11
View File
@@ -25,24 +25,41 @@ import org.jetbrains.kotlin.metadata.deserialization.supertypes
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.deserialization.getClassId
import java.io.File
import java.util.*
/**
* Incremental cache common for JVM and JS
* Incremental cache common for JVM and JS, ClassName type aware
*/
abstract class IncrementalCacheCommon<ClassName>(workingDir: File) : BasicMapsOwner(workingDir) {
interface IncrementalCacheCommon {
val thisWithDependentCaches: Iterable<AbstractIncrementalCache<*>>
fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName>
fun getSubtypesOf(className: FqName): Sequence<FqName>
fun getSourceFileIfClass(fqName: FqName): File?
fun markDirty(removedAndCompiledSources: Collection<File>)
fun clearCacheForRemovedClasses(changesCollector: ChangesCollector)
fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File>
fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl)
fun dump(): String
}
/**
* Incremental cache common for JVM and JS for specifit ClassName type
*/
abstract class AbstractIncrementalCache<ClassName>(workingDir: File) : BasicMapsOwner(workingDir), IncrementalCacheCommon {
companion object {
private val SUBTYPES = "subtypes"
private val SUPERTYPES = "supertypes"
private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
private val COMPLEMENTARY_FILES = "complementary-files"
@JvmStatic protected val SOURCE_TO_CLASSES = "source-to-classes"
@JvmStatic protected val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
}
private val dependents = arrayListOf<IncrementalCacheCommon<ClassName>>()
fun addDependentCache(cache: IncrementalCacheCommon<ClassName>) {
private val dependents = arrayListOf<AbstractIncrementalCache<ClassName>>()
fun addDependentCache(cache: AbstractIncrementalCache<ClassName>) {
dependents.add(cache)
}
val thisWithDependentCaches: Iterable<IncrementalCacheCommon<ClassName>> by lazy {
override val thisWithDependentCaches: Iterable<AbstractIncrementalCache<ClassName>> by lazy {
val result = arrayListOf(this)
result.addAll(dependents)
result
@@ -54,16 +71,24 @@ abstract class IncrementalCacheCommon<ClassName>(workingDir: File) : BasicMapsOw
internal abstract val sourceToClassesMap: AbstractSourceToOutputMap<ClassName>
internal abstract val dirtyOutputClassesMap: AbstractDirtyClassesMap<ClassName>
fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName> =
/**
* A file X is a complementary to a file Y if they contain corresponding expect/actual declarations.
* Complementary files should be compiled together during IC so the compiler does not complain
* about missing parts.
* TODO: provide a better solution (maintain an index of expect/actual declarations akin to IncrementalPackagePartProvider)
*/
private val complementaryFilesMap = registerMap(FilesMap(COMPLEMENTARY_FILES.storageFile))
override fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName> =
files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) }
fun getSubtypesOf(className: FqName): Sequence<FqName> =
override fun getSubtypesOf(className: FqName): Sequence<FqName> =
subtypesMap[className].asSequence()
fun getSourceFileIfClass(fqName: FqName): File? =
override fun getSourceFileIfClass(fqName: FqName): File? =
classFqNameToSourceMap[fqName]
open fun markDirty(removedAndCompiledSources: Collection<File>) {
override fun markDirty(removedAndCompiledSources: Collection<File>) {
for (sourceFile in removedAndCompiledSources) {
val classes = sourceToClassesMap[sourceFile]
classes.forEach {
@@ -90,8 +115,6 @@ abstract class IncrementalCacheCommon<ClassName>(workingDir: File) : BasicMapsOw
classFqNameToSourceMap[child] = srcFile
}
abstract fun clearCacheForRemovedClasses(changesCollector: ChangesCollector)
protected fun removeAllFromClassStorage(removedClasses: Collection<FqName>, changesCollector: ChangesCollector) {
if (removedClasses.isEmpty()) return
@@ -141,4 +164,29 @@ abstract class IncrementalCacheCommon<ClassName>(workingDir: File) : BasicMapsOw
override fun dumpValue(value: String) = value
}
override fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File> {
val complementaryFiles = HashSet<File>()
val filesQueue = ArrayDeque(dirtyFiles)
while (filesQueue.isNotEmpty()) {
val file = filesQueue.pollFirst()
complementaryFilesMap.remove(file).filterTo(filesQueue) { complementaryFiles.add(it) }
}
complementaryFiles.removeAll(dirtyFiles)
return complementaryFiles
}
override fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) {
val actualToExpect = hashMapOf<File, MutableSet<File>>()
for ((expect, actuals) in expectActualTracker.expectToActualMap) {
for (actual in actuals) {
actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect)
}
complementaryFilesMap[expect] = actuals
}
for ((actual, expects) in actualToExpect) {
complementaryFilesMap[actual] = expects
}
}
}
build-common/src/org/jetbrains/kotlin/incremental/CacheVersion.kt
+2 -2
View File
@@ -23,8 +23,8 @@ import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
import java.io.File
import java.io.IOException
private val NORMAL_VERSION = 8
private val DATA_CONTAINER_VERSION = 2
private val NORMAL_VERSION = 9
private val DATA_CONTAINER_VERSION = 3
private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
private val DATA_CONTAINER_VERSION_FILE_NAME = "data-container-format-version.txt"
build-common/src/org/jetbrains/kotlin/incremental/ExpectActualTrackerImpl.kt
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import java.io.File
class ExpectActualTrackerImpl : ExpectActualTracker {
private val expectToActual = hashMapOf<File, MutableSet<File>>()
val expectToActualMap: Map<File, Set<File>>
get() = expectToActual
override fun report(expectedFile: File, actualFile: File) {
expectToActual.getOrPut(expectedFile) { hashSetOf() }.add(actualFile)
}
}
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt
+1 -1
View File
@@ -36,7 +36,7 @@ import java.io.DataInput
import java.io.DataOutput
import java.io.File
open class IncrementalJsCache(cachesDir: File) : IncrementalCacheCommon<FqName>(cachesDir) {
open class IncrementalJsCache(cachesDir: File) : AbstractIncrementalCache<FqName>(cachesDir) {
companion object {
private val TRANSLATION_RESULT_MAP = "translation-result"
private val INLINE_FUNCTIONS = "inline-functions"
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt
+1 -1
View File
@@ -44,7 +44,7 @@ val KOTLIN_CACHE_DIRECTORY_NAME = "kotlin"
open class IncrementalJvmCache(
private val targetDataRoot: File,
targetOutputDir: File?
) : IncrementalCacheCommon<JvmClassName>(File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME)), IncrementalCache {
) : AbstractIncrementalCache<JvmClassName>(File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME)), IncrementalCache {
companion object {
private val PROTO_MAP = "proto"
private val CONSTANTS_MAP = "constants"
build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt
+11 -11
View File
@@ -81,10 +81,10 @@ fun makeCompileServices(
}
fun updateIncrementalCache(
generatedFiles: Iterable<GeneratedFile>,
cache: IncrementalJvmCache,
changesCollector: ChangesCollector,
javaChangesTracker: JavaClassesTrackerImpl?
generatedFiles: Iterable<GeneratedFile>,
cache: IncrementalJvmCache,
changesCollector: ChangesCollector,
javaChangesTracker: JavaClassesTrackerImpl?
) {
for (generatedFile in generatedFiles) {
when {
@@ -119,8 +119,8 @@ data class DirtyData(
)
fun ChangesCollector.getDirtyData(
caches: Iterable<IncrementalCacheCommon<*>>,
reporter: ICReporter
caches: Iterable<IncrementalCacheCommon>,
reporter: ICReporter
): DirtyData {
val dirtyLookupSymbols = HashSet<LookupSymbol>()
val dirtyClassesFqNames = HashSet<FqName>()
@@ -173,10 +173,10 @@ fun mapLookupSymbolsToFiles(
}
fun mapClassesFqNamesToFiles(
caches: Iterable<IncrementalCacheCommon<*>>,
classesFqNames: Iterable<FqName>,
reporter: ICReporter,
excludes: Set<File> = emptySet()
caches: Iterable<IncrementalCacheCommon>,
classesFqNames: Iterable<FqName>,
reporter: ICReporter,
excludes: Set<File> = emptySet()
): Set<File> {
val dirtyFiles = HashSet<File>()
@@ -195,7 +195,7 @@ fun mapClassesFqNamesToFiles(
fun withSubtypes(
typeFqName: FqName,
caches: Iterable<IncrementalCacheCommon<*>>
caches: Iterable<IncrementalCacheCommon>
): Set<FqName> {
val types = LinkedList(listOf(typeFqName))
val subtypes = hashSetOf<FqName>()
build-common/src/org/jetbrains/kotlin/incremental/storage/FilesMap.kt
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File
class FilesMap(storageFile: File)
: BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
operator fun set(sourceFile: File, outputFiles: Collection<File>) {
storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
}
operator fun get(sourceFile: File): Collection<File> =
storage[sourceFile.absolutePath].orEmpty().map(::File)
override fun dumpValue(value: Collection<String>) =
value.dumpCollection()
fun remove(file: File): Collection<File> =
get(file).also { storage.remove(file.absolutePath) }
}
build-common/test/org/jetbrains/kotlin/incremental/testingUtils/incrementalModificationUtils.kt
+9 -3
View File
@@ -67,12 +67,18 @@ fun getModificationsToPerform(
val underscore = fileName.indexOf("_")
if (underscore != -1) {
val module = fileName.substring(0, underscore)
var moduleName = fileName.substring(0, underscore)
var moduleFileName = fileName.substring(underscore + 1)
if (moduleName.all { it.isDigit() }) {
val (moduleName1, moduleFileName1) = moduleFileName.split("_")
moduleName = moduleName1
moduleFileName = moduleFileName1
}
assert(moduleNames != null) { "File name has module prefix, but multi-module environment is absent" }
assert(module in moduleNames!!) { "Module not found for file with prefix: $fileName" }
assert(moduleName in moduleNames!!) { "Module not found for file with prefix: $fileName" }
return Pair(module, fileName.substring(underscore + 1))
return Pair(moduleName, moduleFileName)
}
assert(moduleNames == null) { "Test is multi-module, but file has no module prefix: $fileName" }