diff --git a/build-common/src/org/jetbrains/kotlin/build/BuildMetaInfo.kt b/build-common/src/org/jetbrains/kotlin/build/BuildMetaInfo.kt index 394d113a7ba..5416b5be899 100644 --- a/build-common/src/org/jetbrains/kotlin/build/BuildMetaInfo.kt +++ b/build-common/src/org/jetbrains/kotlin/build/BuildMetaInfo.kt @@ -64,6 +64,9 @@ abstract class BuildMetaInfoFactory(private val metaInfoClass ) } + fun serializeToString(args: CommonCompilerArguments): String = + serializeToString(create(args)) + fun serializeToString(info: T): String = serializeToPlainText(info, metaInfoClass) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/CacheVersion.kt b/build-common/src/org/jetbrains/kotlin/incremental/CacheVersion.kt deleted file mode 100644 index 7b4ad78c691..00000000000 --- a/build-common/src/org/jetbrains/kotlin/incremental/CacheVersion.kt +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.incremental - -import org.jetbrains.annotations.TestOnly -import org.jetbrains.kotlin.config.IncrementalCompilation -import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmBytecodeBinaryVersion -import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion -import java.io.File -import java.io.IOException - -private val NORMAL_VERSION = 10 -private val DATA_CONTAINER_VERSION = 3 - -private val NORMAL_VERSION_FILE_NAME = "format-version.txt" -private val DATA_CONTAINER_VERSION_FILE_NAME = "data-container-format-version.txt" - -class CacheVersion( - private val ownVersion: Int, - private val versionFile: File, - private val whenVersionChanged: CacheVersion.Action, - private val whenTurnedOn: CacheVersion.Action, - private val whenTurnedOff: CacheVersion.Action, - private val isEnabled: Boolean -) { - - private val actualVersion: Int? - get() = try { - versionFile.readText().toInt() - } - catch (e: NumberFormatException) { - null - } - catch (e: IOException) { - null - } - - private val expectedVersion: Int - get() { - val metadata = JvmMetadataVersion.INSTANCE - val bytecode = JvmBytecodeBinaryVersion.INSTANCE - return ownVersion * 1000000 + - bytecode.major * 10000 + bytecode.minor * 100 + - metadata.major * 1000 + metadata.minor - } - - fun checkVersion(): Action = - when (versionFile.exists() to isEnabled) { - true to true -> if (actualVersion != expectedVersion) whenVersionChanged else Action.DO_NOTHING - false to true -> whenTurnedOn - true to false -> whenTurnedOff - else -> Action.DO_NOTHING - } - - fun saveIfNeeded() { - if (!isEnabled) return - - if (!versionFile.parentFile.exists()) { - versionFile.parentFile.mkdirs() - } - - versionFile.writeText(expectedVersion.toString()) - } - - fun clean() { - versionFile.delete() - } - - @get:TestOnly - val formatVersionFile: File - get() = versionFile - - // Order of entries is important, because actions are sorted in KotlinBuilder::checkVersions - enum class Action { - REBUILD_ALL_KOTLIN, - REBUILD_CHUNK, - CLEAN_NORMAL_CACHES, - CLEAN_DATA_CONTAINER, - DO_NOTHING - } -} - -fun normalCacheVersion(dataRoot: File, enabled: Boolean): CacheVersion = - CacheVersion(ownVersion = NORMAL_VERSION, - versionFile = File(dataRoot, NORMAL_VERSION_FILE_NAME), - whenVersionChanged = CacheVersion.Action.REBUILD_CHUNK, - whenTurnedOn = CacheVersion.Action.REBUILD_CHUNK, - whenTurnedOff = CacheVersion.Action.CLEAN_NORMAL_CACHES, - isEnabled = enabled) - -fun dataContainerCacheVersion(dataRoot: File, enabled: Boolean): CacheVersion = - CacheVersion(ownVersion = DATA_CONTAINER_VERSION, - versionFile = File(dataRoot, DATA_CONTAINER_VERSION_FILE_NAME), - whenVersionChanged = CacheVersion.Action.REBUILD_ALL_KOTLIN, - whenTurnedOn = CacheVersion.Action.REBUILD_ALL_KOTLIN, - whenTurnedOff = CacheVersion.Action.CLEAN_DATA_CONTAINER, - isEnabled = enabled) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt index e1ae3d3fe71..832b592caa3 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt @@ -25,6 +25,8 @@ import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.incremental.storage.* +import org.jetbrains.kotlin.incremental.storage.version.clean +import org.jetbrains.kotlin.incremental.storage.version.localCacheVersionManager import org.jetbrains.kotlin.inline.inlineFunctionsJvmNames import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache @@ -43,8 +45,8 @@ import java.util.* val KOTLIN_CACHE_DIRECTORY_NAME = "kotlin" open class IncrementalJvmCache( - private val targetDataRoot: File, - targetOutputDir: File? + private val targetDataRoot: File, + targetOutputDir: File? ) : AbstractIncrementalCache(File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME)), IncrementalCache { companion object { private val PROTO_MAP = "proto" @@ -81,16 +83,16 @@ open class IncrementalJvmCache( // used in gradle @Suppress("unused") fun classesBySources(sources: Iterable): Iterable = - sources.flatMap { sourceToClassesMap[it] } + sources.flatMap { sourceToClassesMap[it] } fun sourceInCache(file: File): Boolean = sourceToClassesMap.contains(file) fun sourcesByInternalName(internalName: String): Collection = - internalNameToSource[internalName] + internalNameToSource[internalName] fun isMultifileFacade(className: JvmClassName): Boolean = - className in multifileFacadeToParts + className in multifileFacadeToParts override fun getClassFilePath(internalClassName: String): String { return toSystemIndependentName(File(outputDir, "$internalClassName.class").canonicalPath) @@ -129,7 +131,7 @@ open class IncrementalJvmCache( } KotlinClassHeader.Kind.MULTIFILE_CLASS -> { val partNames = kotlinClass.classHeader.data?.toList() - ?: throw AssertionError("Multifile class has no parts: ${kotlinClass.className}") + ?: throw AssertionError("Multifile class has no parts: ${kotlinClass.className}") multifileFacadeToParts[className] = partNames // When a class is replaced with a facade with the same name, // the class' proto wouldn't ever be deleted, @@ -177,15 +179,15 @@ open class IncrementalJvmCache( } fun getObsoleteJavaClasses(): Collection = - dirtyOutputClassesMap.getDirtyOutputClasses() - .mapNotNull { - javaSourcesProtoMap[it]?.classId - } + dirtyOutputClassesMap.getDirtyOutputClasses() + .mapNotNull { + javaSourcesProtoMap[it]?.classId + } fun isJavaClassToTrack(classId: ClassId): Boolean { val jvmClassName = JvmClassName.byClassId(classId) return dirtyOutputClassesMap.isDirty(jvmClassName) || - jvmClassName !in javaSourcesProtoMap + jvmClassName !in javaSourcesProtoMap } fun isJavaClassAlreadyInCache(classId: ClassId): Boolean { @@ -210,8 +212,7 @@ open class IncrementalJvmCache( if (notRemovedParts.isEmpty()) { multifileFacadeToParts.remove(facade) - } - else { + } else { multifileFacadeToParts[facade] = notRemovedParts } } @@ -265,7 +266,7 @@ open class IncrementalJvmCache( override fun clean() { super.clean() - normalCacheVersion(targetDataRoot, IncrementalCompilation.isEnabledForJvm()).clean() + localCacheVersionManager(targetDataRoot, IncrementalCompilation.isEnabledForJvm()).clean() } private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { @@ -290,9 +291,11 @@ open class IncrementalJvmCache( val key = kotlinClass.className.internalName val oldData = storage[key] - val newData = ProtoMapValue(header.kind != KotlinClassHeader.Kind.CLASS, - BitEncoding.decodeBytes(header.data!!), - header.strings!!) + val newData = ProtoMapValue( + header.kind != KotlinClassHeader.Kind.CLASS, + BitEncoding.decodeBytes(header.data!!), + header.strings!! + ) storage[key] = newData val packageFqName = kotlinClass.className.packageFqName @@ -300,10 +303,10 @@ open class IncrementalJvmCache( } operator fun contains(className: JvmClassName): Boolean = - className.internalName in storage + className.internalName in storage operator fun get(className: JvmClassName): ProtoMapValue? = - storage[className.internalName] + storage[className.internalName] fun remove(className: JvmClassName, changesCollector: ChangesCollector) { val key = className.internalName @@ -319,15 +322,16 @@ open class IncrementalJvmCache( } } - private inner class JavaSourcesProtoMap(storageFile: File) : BasicStringMap(storageFile, JavaClassProtoMapValueExternalizer) { + private inner class JavaSourcesProtoMap(storageFile: File) : + BasicStringMap(storageFile, JavaClassProtoMapValueExternalizer) { fun process(jvmClassName: JvmClassName, newData: SerializedJavaClass, changesCollector: ChangesCollector) { val key = jvmClassName.internalName val oldData = storage[key] storage[key] = newData changesCollector.collectProtoChanges( - oldData?.toProtoData(), newData.toProtoData(), - collectAllMembersForNewClass = true + oldData?.toProtoData(), newData.toProtoData(), + collectAllMembersForNewClass = true ) } @@ -340,13 +344,13 @@ open class IncrementalJvmCache( } operator fun get(className: JvmClassName): SerializedJavaClass? = - storage[className.internalName] + storage[className.internalName] operator fun contains(className: JvmClassName): Boolean = - className.internalName in storage + className.internalName in storage override fun dumpValue(value: SerializedJavaClass): String = - java.lang.Long.toHexString(value.proto.toByteArray().md5()) + java.lang.Long.toHexString(value.proto.toByteArray().md5()) } // todo: reuse code with InlineFunctionsMap? @@ -368,7 +372,7 @@ open class IncrementalJvmCache( } operator fun contains(className: JvmClassName): Boolean = - className.internalName in storage + className.internalName in storage fun process(kotlinClass: LocalFileKotlinClass, changesCollector: ChangesCollector) { val key = kotlinClass.className.internalName @@ -377,8 +381,7 @@ open class IncrementalJvmCache( val newMap = getConstantsMap(kotlinClass.fileContents) if (newMap.isNotEmpty()) { storage[key] = newMap - } - else { + } else { storage.remove(key) } @@ -392,7 +395,7 @@ open class IncrementalJvmCache( } override fun dumpValue(value: Map): String = - value.dumpMap(Any::toString) + value.dumpMap(Any::toString) } private inner class PackagePartMap(storageFile: File) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { @@ -405,21 +408,22 @@ open class IncrementalJvmCache( } fun isPackagePart(className: JvmClassName): Boolean = - className.internalName in storage + className.internalName in storage override fun dumpValue(value: Boolean) = "" } - private inner class MultifileClassFacadeMap(storageFile: File) : BasicStringMap>(storageFile, StringCollectionExternalizer) { + private inner class MultifileClassFacadeMap(storageFile: File) : + BasicStringMap>(storageFile, StringCollectionExternalizer) { operator fun set(className: JvmClassName, partNames: Collection) { storage[className.internalName] = partNames } operator fun get(className: JvmClassName): Collection? = - storage[className.internalName] + storage[className.internalName] operator fun contains(className: JvmClassName): Boolean = - className.internalName in storage + className.internalName in storage fun remove(className: JvmClassName) { storage.remove(className.internalName) @@ -428,13 +432,14 @@ open class IncrementalJvmCache( override fun dumpValue(value: Collection): String = value.dumpCollection() } - private inner class MultifileClassPartMap(storageFile: File) : BasicStringMap(storageFile, EnumeratorStringDescriptor.INSTANCE) { + private inner class MultifileClassPartMap(storageFile: File) : + BasicStringMap(storageFile, EnumeratorStringDescriptor.INSTANCE) { fun set(partName: String, facadeName: String) { storage[partName] = facadeName } fun get(partName: JvmClassName): String? = - storage[partName.internalName] + storage[partName.internalName] fun remove(className: JvmClassName) { storage.remove(className.internalName) @@ -443,20 +448,21 @@ open class IncrementalJvmCache( override fun dumpValue(value: String): String = value } - inner class InternalNameToSourcesMap(storageFile: File) : BasicStringMap>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) { + inner class InternalNameToSourcesMap(storageFile: File) : + BasicStringMap>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) { operator fun set(internalName: String, sourceFiles: Iterable) { storage[internalName] = sourceFiles.map { it.canonicalPath } } operator fun get(internalName: String): Collection = - (storage[internalName] ?: emptyList()).map(::File) + (storage[internalName] ?: emptyList()).map(::File) fun remove(internalName: String) { storage.remove(internalName) } override fun dumpValue(value: Collection): String = - value.dumpCollection() + value.dumpCollection() } private fun addToClassStorage(kotlinClass: LocalFileKotlinClass, srcFile: File) { @@ -464,7 +470,8 @@ open class IncrementalJvmCache( addToClassStorage(proto, nameResolver, srcFile) } - private inner class InlineFunctionsMap(storageFile: File) : BasicStringMap>(storageFile, StringToLongMapExternalizer) { + private inner class InlineFunctionsMap(storageFile: File) : + BasicStringMap>(storageFile, StringToLongMapExternalizer) { private fun getInlineFunctionsMap(header: KotlinClassHeader, bytes: ByteArray): Map { val inlineFunctions = inlineFunctionsJvmNames(header) if (inlineFunctions.isEmpty()) return emptyMap() @@ -472,7 +479,13 @@ open class IncrementalJvmCache( val result = HashMap() ClassReader(bytes).accept(object : ClassVisitor(Opcodes.ASM5) { - override fun visitMethod(access: Int, name: String, desc: String, signature: String?, exceptions: Array?): MethodVisitor? { + override fun visitMethod( + access: Int, + name: String, + desc: String, + signature: String?, + exceptions: Array? + ): MethodVisitor? { val dummyClassWriter = ClassWriter(Opcodes.ASM5) return object : MethodVisitor(Opcodes.ASM5, dummyClassWriter.visitMethod(0, name, desc, null, exceptions)) { @@ -499,30 +512,35 @@ open class IncrementalJvmCache( val newMap = getInlineFunctionsMap(kotlinClass.classHeader, kotlinClass.fileContents) if (newMap.isNotEmpty()) { storage[key] = newMap - } - else { + } else { storage.remove(key) } for (fn in oldMap.keys + newMap.keys) { - changesCollector.collectMemberIfValueWasChanged(kotlinClass.scopeFqName(), functionNameBySignature(fn), oldMap[fn], newMap[fn]) + changesCollector.collectMemberIfValueWasChanged( + kotlinClass.scopeFqName(), + functionNameBySignature(fn), + oldMap[fn], + newMap[fn] + ) } } // TODO get name in better way instead of using substringBefore private fun functionNameBySignature(signature: String): String = - signature.substringBefore("(") + signature.substringBefore("(") fun remove(className: JvmClassName) { storage.remove(className.internalName) } override fun dumpValue(value: Map): String = - value.dumpMap { java.lang.Long.toHexString(it) } + value.dumpMap { java.lang.Long.toHexString(it) } } } -private object PathCollectionExternalizer : CollectionExternalizer(PathStringDescriptor, { THashSet(FileUtil.PATH_HASHING_STRATEGY) }) +private object PathCollectionExternalizer : + CollectionExternalizer(PathStringDescriptor, { THashSet(FileUtil.PATH_HASHING_STRATEGY) }) sealed class ChangeInfo(val fqName: FqName) { open class MembersChanged(fqName: FqName, val names: Collection) : ChangeInfo(fqName) { @@ -542,10 +560,10 @@ sealed class ChangeInfo(val fqName: FqName) { } private fun LocalFileKotlinClass.scopeFqName() = - when (classHeader.kind) { - KotlinClassHeader.Kind.CLASS -> className.fqNameForClassNameWithoutDollars - else -> className.packageFqName - } + when (classHeader.kind) { + KotlinClassHeader.Kind.CLASS -> className.fqNameForClassNameWithoutDollars + else -> className.packageFqName + } fun ByteArray.md5(): Long { val d = MessageDigest.getInstance("MD5").digest(this)!! @@ -557,23 +575,24 @@ fun ByteArray.md5(): Long { or ((d[5].toLong() and 0xFFL) shl 40) or ((d[6].toLong() and 0xFFL) shl 48) or ((d[7].toLong() and 0xFFL) shl 56) - ) + ) } @TestOnly -fun , V> Map.dumpMap(dumpValue: (V)->String): String = - buildString { - append("{") - for (key in keys.sorted()) { - if (length != 1) { - append(", ") - } - - val value = get(key)?.let(dumpValue) ?: "null" - append("$key -> $value") +fun , V> Map.dumpMap(dumpValue: (V) -> String): String = + buildString { + append("{") + for (key in keys.sorted()) { + if (length != 1) { + append(", ") } - append("}") - } -@TestOnly fun > Collection.dumpCollection(): String = - "[${sorted().joinToString(", ", transform = Any::toString)}]" + val value = get(key)?.let(dumpValue) ?: "null" + append("$key -> $value") + } + append("}") + } + +@TestOnly +fun > Collection.dumpCollection(): String = + "[${sorted().joinToString(", ", transform = Any::toString)}]" diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesDiff.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesDiff.kt new file mode 100644 index 00000000000..fab3d10f29a --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesDiff.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +/** + * Diff between actual and expected cache attributes. + * [status] are calculated based on this diff (see [CacheStatus]). + * Based on that [status] system may perform required actions (i.e. rebuild something, clearing caches, etc...). + * + * [CacheAttributesDiff] can be used to cache current attribute values and as facade for version operations. + */ +data class CacheAttributesDiff( + val manager: CacheAttributesManager, + val actual: Attrs?, + val expected: Attrs? +) { + val status: CacheStatus + get() = + if (expected != null) { + if (actual != null && manager.isCompatible(actual, expected)) CacheStatus.VALID + else CacheStatus.INVALID + } else { + if (actual != null) CacheStatus.SHOULD_BE_CLEARED + else CacheStatus.CLEARED + } + + fun saveExpectedIfNeeded() { + if (expected != actual) manager.writeActualVersion(expected) + } + + override fun toString(): String { + return "$status: actual=$actual -> expected=$expected" + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesManager.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesManager.kt new file mode 100644 index 00000000000..54d176761cb --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheAttributesManager.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +/** + * Manages cache attributes values. + * + * Attribute values can be loaded by calling [loadActual]. + * Based on loaded actual and fixed [expected] values [CacheAttributesDiff] can be constructed which can calculate [CacheStatus]. + * Build system may perform required actions based on that (i.e. rebuild something, clearing caches, etc...). + * + * [CacheAttributesDiff] can be used to cache current attribute values and then can be used as facade for cache version operations. + */ +interface CacheAttributesManager { + /** + * Cache attribute values expected by the current version of build system and compiler. + * `null` means that cache is not required (incremental compilation is disabled). + */ + val expected: Attrs? + + /** + * Load actual cache attribute values. + * `null` means that cache is not yet created. + * + * This is internal operation that should be implemented by particular implementation of CacheAttributesManager. + * Consider using `loadDiff().actual` for getting actual values. + */ + fun loadActual(): Attrs? + + /** + * Write [values] as cache attributes for next build execution. + * + * This is internal operation that should be implemented by particular implementation of CacheAttributesManager. + * Consider using `loadDiff().saveExpectedIfNeeded()` for saving attributes values for next build. + */ + fun writeActualVersion(values: Attrs?) + + /** + * Check if cache with [actual] attributes values can be used when [expected] attributes are required. + */ + fun isCompatible(actual: Attrs, expected: Attrs): Boolean = actual == expected +} + +fun CacheAttributesManager.loadDiff( + actual: Attrs? = this.loadActual(), + expected: Attrs? = this.expected +) = CacheAttributesDiff(this, actual, expected) + +fun CacheAttributesManager.loadAndCheckStatus() = + loadDiff().status + +/** + * This method is kept only for compatibility. + * Save [expected] cache attributes values if it is enabled and not equals to [actual]. + */ +@Deprecated( + message = "Consider using `this.loadDiff().saveExpectedIfNeeded()` and cache `loadDiff()` result.", + replaceWith = ReplaceWith("loadDiff().saveExpectedIfNeeded()") +) +fun CacheAttributesManager.saveIfNeeded( + actual: Attrs? = this.loadActual(), + expected: Attrs = this.expected + ?: error("To save disabled cache status [delete] should be called (this behavior is kept for compatibility)") +) = loadDiff(actual, expected).saveExpectedIfNeeded() + +/** + * This method is kept only for compatibility. + * Delete actual cache attributes values if it existed. + */ +@Deprecated( + message = "Consider using `this.loadDiff().saveExpectedIfNeeded()` and cache `loadDiff()` result.", + replaceWith = ReplaceWith("writeActualVersion(null)") +) +fun CacheAttributesManager<*>.clean() { + writeActualVersion(null) +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheStatus.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheStatus.kt new file mode 100644 index 00000000000..2463767554e --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheStatus.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +/** + * Status that is used by system to perform required actions (i.e. rebuild something, clearing caches, etc...). + */ +enum class CacheStatus { + /** + * Cache is valid and ready to use. + */ + VALID, + + /** + * Cache is not exists or have outdated versions and/or other attributes. + */ + INVALID, + + /** + * Cache is exists, but not required anymore. + */ + SHOULD_BE_CLEARED, + + /** + * Cache is not exists and not required. + */ + CLEARED +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheVersionManager.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheVersionManager.kt new file mode 100644 index 00000000000..2fdc91155e0 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/CacheVersionManager.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +import org.jetbrains.annotations.TestOnly +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmBytecodeBinaryVersion +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion +import java.io.File +import java.io.IOException + +/** + * Manages files with actual version [loadActual] and provides expected version [expected]. + * Based on that actual and expected versions [CacheStatus] can be calculated. + * This can be done by constructing [CacheAttributesDiff] and calling [CacheAttributesDiff.status]. + * Based on that status system may perform required actions (i.e. rebuild something, clearing caches, etc...). + */ +class CacheVersionManager( + private val versionFile: File, + expectedOwnVersion: Int? +) : CacheAttributesManager { + override val expected: CacheVersion? = + if (expectedOwnVersion == null) null + else { + val metadata = JvmMetadataVersion.INSTANCE + val bytecode = JvmBytecodeBinaryVersion.INSTANCE + + CacheVersion( + expectedOwnVersion * 1000000 + + bytecode.major * 10000 + bytecode.minor * 100 + + metadata.major * 1000 + metadata.minor + ) + } + + override fun loadActual(): CacheVersion? = + if (!versionFile.exists()) null + else try { + CacheVersion(versionFile.readText().toInt()) + } catch (e: NumberFormatException) { + null + } catch (e: IOException) { + null + } + + override fun writeActualVersion(values: CacheVersion?) { + if (values == null) versionFile.delete() + else { + versionFile.parentFile.mkdirs() + versionFile.writeText(values.version.toString()) + } + } + + @get:TestOnly + val versionFileForTesting: File + get() = versionFile +} + +data class CacheVersion(val version: Int) \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/local.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/local.kt new file mode 100644 index 00000000000..344abd0c542 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/local.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +import java.io.File + +private val NORMAL_VERSION = 9 +private val NORMAL_VERSION_FILE_NAME = "format-version.txt" + +fun localCacheVersionManager(dataRoot: File, isCachesEnabled: Boolean) = + CacheVersionManager( + File(dataRoot, NORMAL_VERSION_FILE_NAME), + if (isCachesEnabled) NORMAL_VERSION else null + ) \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/version/lookups.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/lookups.kt new file mode 100644 index 00000000000..bd8fdd78c04 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/version/lookups.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.storage.version + +import java.io.File + +private val DATA_CONTAINER_VERSION_FILE_NAME = "data-container-format-version.txt" +private val DATA_CONTAINER_VERSION = 3 + +fun lookupsCacheVersionManager(dataRoot: File, isEnabled: Boolean) = + CacheVersionManager( + File(dataRoot, DATA_CONTAINER_VERSION_FILE_NAME), + if (isEnabled) DATA_CONTAINER_VERSION else null + ) + +fun readLookupsCacheStatus(dataRoot: File, isEnabled: Boolean): CacheStatus = + lookupsCacheVersionManager(dataRoot, isEnabled).loadAndCheckStatus() \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index 67f6385e751..b71283100d9 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -83,7 +83,7 @@ interface CompilerSelector { } interface EventManager { - fun onCompilationFinished(f : () -> Unit) + fun onCompilationFinished(f: () -> Unit) } private class EventManagerImpl : EventManager { @@ -99,14 +99,14 @@ private class EventManagerImpl : EventManager { } class CompileServiceImpl( - val registry: Registry, - val compiler: CompilerSelector, - val compilerId: CompilerId, - val daemonOptions: DaemonOptions, - val daemonJVMOptions: DaemonJVMOptions, - val port: Int, - val timer: Timer, - val onShutdown: () -> Unit + val registry: Registry, + val compiler: CompilerSelector, + val compilerId: CompilerId, + val daemonOptions: DaemonOptions, + val daemonJVMOptions: DaemonJVMOptions, + val port: Int, + val timer: Timer, + val onShutdown: () -> Unit ) : CompileService { private val log by lazy { Logger.getLogger("compiler") } @@ -116,8 +116,14 @@ class CompileServiceImpl( } // wrapped in a class to encapsulate alive check logic - private class ClientOrSessionProxy(val aliveFlagPath: String?, val data: T? = null, private var disposable: Disposable? = null) { - val isAlive: Boolean get() = aliveFlagPath?.let { File(it).exists() } ?: true // assuming that if no file was given, the client is alive + private class ClientOrSessionProxy( + val aliveFlagPath: String?, + val data: T? = null, + private var disposable: Disposable? = null + ) { + val isAlive: Boolean + get() = aliveFlagPath?.let { File(it).exists() } ?: true // assuming that if no file was given, the client is alive + fun dispose() { disposable?.let { Disposer.dispose(it) @@ -132,7 +138,8 @@ class CompileServiceImpl( enum class Aliveness { // !!! ordering of values is used in state comparison - Dying, LastSession, Alive + Dying, + LastSession, Alive } private class SessionsContainer { @@ -143,7 +150,7 @@ class CompileServiceImpl( val lastSessionId get() = sessionsIdCounter.get() - fun leaseSession(session: ClientOrSessionProxy): Int = lock.write { + fun leaseSession(session: ClientOrSessionProxy): Int = lock.write { val newId = getValidId(sessionsIdCounter) { it != CompileService.NO_SESSION && !sessions.containsKey(it) } @@ -203,18 +210,22 @@ class CompileServiceImpl( clientProxies.mapNotNull { it.aliveFlagPath } } - fun cleanDeadClients(): Boolean = clientProxies.cleanMatching(clientsLock, { !it.isAlive }, { if (clientProxies.remove(it)) it.dispose() }) + fun cleanDeadClients(): Boolean = + clientProxies.cleanMatching(clientsLock, { !it.isAlive }, { if (clientProxies.remove(it)) it.dispose() }) } private fun Int.toAlivenessName(): String = - try { - Aliveness.values()[this].name - } - catch (_: Throwable) { - "invalid($this)" - } + try { + Aliveness.values()[this].name + } catch (_: Throwable) { + "invalid($this)" + } - private inline fun Iterable.cleanMatching(lock: ReentrantReadWriteLock, crossinline pred: (T) -> Boolean, crossinline clean: (T) -> Unit): Boolean { + private inline fun Iterable.cleanMatching( + lock: ReentrantReadWriteLock, + crossinline pred: (T) -> Boolean, + crossinline clean: (T) -> Unit + ): Boolean { var anyDead = false lock.read { val toRemove = filter(pred) @@ -228,7 +239,8 @@ class CompileServiceImpl( return anyDead } - @Volatile private var _lastUsedSeconds = nowSeconds() + @Volatile + private var _lastUsedSeconds = nowSeconds() val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds private val rwlock = ReentrantReadWriteLock() @@ -238,10 +250,14 @@ class CompileServiceImpl( init { val runFileDir = File(daemonOptions.runFilesPathOrDefault) runFileDir.mkdirs() - runFile = File(runFileDir, - makeRunFilenameString(timestamp = "%tFT% = ifAlive(minAliveness = Aliveness.Alive) { CompileService.CallResult.Good( - state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath)).apply { - log.info("leased a new session $this, session alive file: $aliveFlagPath") - }) + state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath)).apply { + log.info("leased a new session $this, session alive file: $aliveFlagPath") + }) } @@ -301,12 +317,12 @@ class CompileServiceImpl( } override fun checkCompilerId(expectedCompilerId: CompilerId): Boolean = - (compilerId.compilerVersion.isEmpty() || compilerId.compilerVersion == expectedCompilerId.compilerVersion) && - (compilerId.compilerClasspath.all { expectedCompilerId.compilerClasspath.contains(it) }) && - !classpathWatcher.isChanged + (compilerId.compilerVersion.isEmpty() || compilerId.compilerVersion == expectedCompilerId.compilerVersion) && + (compilerId.compilerClasspath.all { expectedCompilerId.compilerClasspath.contains(it) }) && + !classpathWatcher.isChanged override fun getUsedMemory(): CompileService.CallResult = - ifAlive { CompileService.CallResult.Good(usedMemory(withGC = true)) } + ifAlive { CompileService.CallResult.Good(usedMemory(withGC = true)) } override fun shutdown(): CompileService.CallResult = ifAliveExclusive(minAliveness = Aliveness.LastSession) { shutdownWithDelay() @@ -324,37 +340,55 @@ class CompileServiceImpl( CompileService.CallResult.Good(res) } - override fun remoteCompile(sessionId: Int, - targetPlatform: CompileService.TargetPlatform, - args: Array, - servicesFacade: CompilerCallbackServicesFacade, - compilerOutputStream: RemoteOutputStream, - outputFormat: CompileService.OutputFormat, - serviceOutputStream: RemoteOutputStream, - operationsTracer: RemoteOperationsTracer? + override fun remoteCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + outputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? ): CompileService.CallResult = - doCompile(sessionId, args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> - when (outputFormat) { - CompileService.OutputFormat.PLAIN -> compiler[targetPlatform].exec(printStream, *args) - CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, eventManager, profiler), *args) - } + doCompile(sessionId, args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler[targetPlatform].exec(printStream, *args) + CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml( + printStream, + createCompileServices( + servicesFacade, + eventManager, + profiler + ), + *args + ) } + } - override fun remoteIncrementalCompile(sessionId: Int, - targetPlatform: CompileService.TargetPlatform, - args: Array, - servicesFacade: CompilerCallbackServicesFacade, - compilerOutputStream: RemoteOutputStream, - compilerOutputFormat: CompileService.OutputFormat, - serviceOutputStream: RemoteOutputStream, - operationsTracer: RemoteOperationsTracer? + override fun remoteIncrementalCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + compilerOutputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? ): CompileService.CallResult = - doCompile(sessionId, args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> - when (compilerOutputFormat) { - CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, eventManager, profiler), *args) - } + doCompile(sessionId, args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> + when (compilerOutputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml( + printStream, + createCompileServices( + servicesFacade, + eventManager, + profiler + ), + *args + ) } + } override fun classesFqNamesByFiles( sessionId: Int, sourceFiles: Set @@ -366,11 +400,11 @@ class CompileServiceImpl( } override fun compile( - sessionId: Int, - compilerArguments: Array, - compilationOptions: CompilationOptions, - servicesFacade: CompilerServicesFacadeBase, - compilationResults: CompilationResults? + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + compilationResults: CompilationResults? ): CompileService.CallResult = ifAlive { withValidClientOrSessionProxy(sessionId) { val messageCollector = CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions) @@ -391,8 +425,7 @@ class CompileServiceImpl( if (argumentParseError != null) { messageCollector.report(CompilerMessageSeverity.ERROR, argumentParseError) CompileService.CallResult.Good(ExitCode.COMPILATION_ERROR.code) - } - else when (compilationOptions.compilerMode) { + } else when (compilationOptions.compilerMode) { CompilerMode.JPS_COMPILER -> { val jpsServicesFacade = servicesFacade as JpsCompilerServicesFacade @@ -418,8 +451,10 @@ class CompileServiceImpl( withIC { doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - execIncrementalCompiler(k2jvmArgs, gradleIncrementalArgs, gradleIncrementalServicesFacade, compilationResults!!, - messageCollector, daemonReporter) + execIncrementalCompiler( + k2jvmArgs, gradleIncrementalArgs, gradleIncrementalServicesFacade, compilationResults!!, + messageCollector, daemonReporter + ) } } } @@ -428,7 +463,13 @@ class CompileServiceImpl( withJsIC { doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - execJsIncrementalCompiler(k2jsArgs, gradleIncrementalArgs, gradleIncrementalServicesFacade, compilationResults!!, messageCollector) + execJsIncrementalCompiler( + k2jsArgs, + gradleIncrementalArgs, + gradleIncrementalServicesFacade, + compilationResults!!, + messageCollector + ) } } } @@ -451,10 +492,9 @@ class CompileServiceImpl( val allKotlinFiles = arrayListOf() val freeArgsWithoutKotlinFiles = arrayListOf() args.freeArgs.forEach { - if (it.endsWith(".kt") && File(it).exists()) { + if (it.endsWith(".kt") && File(it).exists()) { allKotlinFiles.add(File(it)) - } - else { + } else { freeArgsWithoutKotlinFiles.add(it) } } @@ -464,21 +504,22 @@ class CompileServiceImpl( val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) - } - else { + } else { ChangedFiles.Unknown() } val workingDir = incrementalCompilationOptions.workingDir - val versions = commonCacheVersions(workingDir, enabled = true) + - customCacheVersion(incrementalCompilationOptions.customCacheVersion, - incrementalCompilationOptions.customCacheVersionFileName, - workingDir, - enabled = true) + val versionManagers = commonCacheVersionsManagers(workingDir, enabled = true) + + customCacheVersionManager( + incrementalCompilationOptions.customCacheVersion, + incrementalCompilationOptions.customCacheVersionFileName, + workingDir, + enabled = true + ) val modulesApiHistory = ModulesApiHistoryJs(incrementalCompilationOptions.modulesInfo) val compiler = IncrementalJsCompilerRunner( workingDir = workingDir, - cacheVersions = versions, + cachesVersionManagers = versionManagers, reporter = reporter, buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, modulesApiHistory = modulesApiHistory @@ -487,12 +528,12 @@ class CompileServiceImpl( } private fun execIncrementalCompiler( - k2jvmArgs: K2JVMCompilerArguments, - incrementalCompilationOptions: IncrementalCompilationOptions, - servicesFacade: IncrementalCompilerServicesFacade, - compilationResults: CompilationResults, - compilerMessageCollector: MessageCollector, - daemonMessageReporter: DaemonMessageReporter + k2jvmArgs: K2JVMCompilerArguments, + incrementalCompilationOptions: IncrementalCompilationOptions, + servicesFacade: IncrementalCompilerServicesFacade, + compilationResults: CompilationResults, + compilerMessageCollector: MessageCollector, + daemonMessageReporter: DaemonMessageReporter ): ExitCode { val reporter = RemoteICReporter(servicesFacade, compilationResults, incrementalCompilationOptions) @@ -522,17 +563,18 @@ class CompileServiceImpl( val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) - } - else { + } else { ChangedFiles.Unknown() } val workingDir = incrementalCompilationOptions.workingDir - val versions = commonCacheVersions(workingDir, enabled = true) + - customCacheVersion(incrementalCompilationOptions.customCacheVersion, - incrementalCompilationOptions.customCacheVersionFileName, - workingDir, - enabled = true) + val versions = commonCacheVersionsManagers(workingDir, enabled = true) + + customCacheVersionManager( + incrementalCompilationOptions.customCacheVersion, + incrementalCompilationOptions.customCacheVersionFileName, + workingDir, + enabled = true + ) val modulesApiHistory = incrementalCompilationOptions.run { if (!multiModuleICSettings.useModuleDetection) { @@ -556,27 +598,34 @@ class CompileServiceImpl( } override fun leaseReplSession( - aliveFlagPath: String?, - targetPlatform: CompileService.TargetPlatform, - servicesFacade: CompilerCallbackServicesFacade, - templateClasspath: List, - templateClassName: String, - scriptArgs: Array?, - scriptArgsTypes: Array>?, - compilerMessagesOutputStream: RemoteOutputStream, - evalOutputStream: RemoteOutputStream?, - evalErrorStream: RemoteOutputStream?, - evalInputStream: RemoteInputStream?, - operationsTracer: RemoteOperationsTracer? + aliveFlagPath: String?, + targetPlatform: CompileService.TargetPlatform, + servicesFacade: CompilerCallbackServicesFacade, + templateClasspath: List, + templateClassName: String, + scriptArgs: Array?, + scriptArgsTypes: Array>?, + compilerMessagesOutputStream: RemoteOutputStream, + evalOutputStream: RemoteOutputStream?, + evalErrorStream: RemoteOutputStream?, + evalInputStream: RemoteInputStream?, + operationsTracer: RemoteOperationsTracer? ): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { if (targetPlatform != CompileService.TargetPlatform.JVM) CompileService.CallResult.Error("Sorry, only JVM target platform is supported now") else { val disposable = Disposer.newDisposable() - val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerMessagesOutputStream, DummyProfiler()), REMOTE_STREAM_BUFFER_SIZE)) + val compilerMessagesStream = PrintStream( + BufferedOutputStream( + RemoteOutputStreamClient(compilerMessagesOutputStream, DummyProfiler()), + REMOTE_STREAM_BUFFER_SIZE + ) + ) val messageCollector = KeepFirstErrorMessageCollector(compilerMessagesStream) - val repl = KotlinJvmReplService(disposable, port, templateClasspath, templateClassName, - messageCollector, operationsTracer) + val repl = KotlinJvmReplService( + disposable, port, templateClasspath, templateClassName, + messageCollector, operationsTracer + ) val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable)) CompileService.CallResult.Good(sessionId) @@ -587,42 +636,49 @@ class CompileServiceImpl( override fun releaseReplSession(sessionId: Int): CompileService.CallResult = releaseCompileSession(sessionId) override fun remoteReplLineCheck(sessionId: Int, codeLine: ReplCodeLine): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - withValidRepl(sessionId) { - CompileService.CallResult.Good(check(codeLine)) - } + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + CompileService.CallResult.Good(check(codeLine)) } + } - override fun remoteReplLineCompile(sessionId: Int, codeLine: ReplCodeLine, history: List?): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - withValidRepl(sessionId) { - CompileService.CallResult.Good(compile(codeLine, history)) - } + override fun remoteReplLineCompile( + sessionId: Int, + codeLine: ReplCodeLine, + history: List? + ): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + CompileService.CallResult.Good(compile(codeLine, history)) } + } override fun remoteReplLineEval( - sessionId: Int, - codeLine: ReplCodeLine, - history: List? + sessionId: Int, + codeLine: ReplCodeLine, + history: List? ): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - CompileService.CallResult.Error("Eval on daemon is not supported") - } + ifAlive(minAliveness = Aliveness.Alive) { + CompileService.CallResult.Error("Eval on daemon is not supported") + } - override fun leaseReplSession(aliveFlagPath: String?, - compilerArguments: Array, - compilationOptions: CompilationOptions, - servicesFacade: CompilerServicesFacadeBase, - templateClasspath: List, - templateClassName: String - ): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { + override fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + templateClasspath: List, + templateClassName: String + ): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { if (compilationOptions.targetPlatform != CompileService.TargetPlatform.JVM) CompileService.CallResult.Error("Sorry, only JVM target platform is supported now") else { val disposable = Disposer.newDisposable() val messageCollector = CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions) - val repl = KotlinJvmReplService(disposable, port, templateClasspath, templateClassName, - messageCollector, null) + val repl = KotlinJvmReplService( + disposable, port, templateClasspath, templateClassName, + messageCollector, null + ) val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable)) CompileService.CallResult.Good(sessionId) @@ -630,29 +686,29 @@ class CompileServiceImpl( } override fun replCreateState(sessionId: Int): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - withValidRepl(sessionId) { - CompileService.CallResult.Good(createRemoteState(port)) - } + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + CompileService.CallResult.Good(createRemoteState(port)) } + } override fun replCheck(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - withValidRepl(sessionId) { - withValidReplState(replStateId) { state -> - check(state, codeLine) - } + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + withValidReplState(replStateId) { state -> + check(state, codeLine) } } + } override fun replCompile(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine): CompileService.CallResult = - ifAlive(minAliveness = Aliveness.Alive) { - withValidRepl(sessionId) { - withValidReplState(replStateId) { state -> - compile(state, codeLine) - } + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + withValidReplState(replStateId) { state -> + compile(state, codeLine) } } + } // ----------------------------------------------------------------------- // internal implementation stuff @@ -675,13 +731,17 @@ class CompileServiceImpl( try { // cleanup for the case of incorrect restart and many other situations UnicastRemoteObject.unexportObject(this, false) - } - catch (e: NoSuchObjectException) { + } catch (e: NoSuchObjectException) { // ignoring if object already exported } - val stub = UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) as CompileService - registry.rebind (COMPILER_SERVICE_RMI_NAME, stub) + val stub = UnicastRemoteObject.exportObject( + this, + port, + LoopbackNetworkInterface.clientLoopbackSocketFactory, + LoopbackNetworkInterface.serverLoopbackSocketFactory + ) as CompileService + registry.rebind(COMPILER_SERVICE_RMI_NAME, stub) timer.schedule(10) { exceptionLoggingTimerThread { initiateElections() } @@ -697,8 +757,7 @@ class CompileServiceImpl( private inline fun exceptionLoggingTimerThread(body: () -> Unit) { try { body() - } - catch (e: Throwable) { + } catch (e: Throwable) { System.err.println("Exception in timer thread: " + e.message) e.printStackTrace(System.err) log.log(Level.SEVERE, "Exception in timer thread", e) @@ -768,12 +827,22 @@ class CompileServiceImpl( ifAliveUnit { log.info("initiate elections") - val aliveWithOpts = walkDaemons(File(daemonOptions.runFilesPathOrDefault), compilerId, runFile, filter = { _, p -> p != port }, report = { _, msg -> log.info(msg) }).toList() - val comparator = compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions }) + val aliveWithOpts = walkDaemons( + File(daemonOptions.runFilesPathOrDefault), + compilerId, + runFile, + filter = { _, p -> p != port }, + report = { _, msg -> log.info(msg) }).toList() + val comparator = + compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions }) .thenBy(FileAgeComparator()) { it.runFile } aliveWithOpts.maxWith(comparator)?.let { bestDaemonWithMetadata -> val fattestOpts = bestDaemonWithMetadata.jvmOptions - if (fattestOpts memorywiseFitsInto daemonJVMOptions && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) < 0 ) { + if (fattestOpts memorywiseFitsInto daemonJVMOptions && FileAgeComparator().compare( + bestDaemonWithMetadata.runFile, + runFile + ) < 0 + ) { // all others are smaller that me, take overs' clients and shut them down log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE lower prio, taking clients from them and schedule them to shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") aliveWithOpts.forEach { (daemon, runFile, _) -> @@ -782,8 +851,7 @@ class CompileServiceImpl( it.get().forEach { clientAliveFile -> registerClient(clientAliveFile) } } daemon.scheduleShutdown(true) - } - catch (e: Throwable) { + } catch (e: Throwable) { log.info("Cannot connect to a daemon, assuming dying ('${runFile.canonicalPath}'): ${e.message}") } } @@ -803,15 +871,18 @@ class CompileServiceImpl( // B performs election: (1) is false because neither A nor C does not fit into B, (2) is false because B does not fit into neither A nor C. // C performs election: (1) is false because B is better than A and B does not fit into C, (2) is false C does not fit into neither A nor B. // Result: all daemons are alive and well. - else if (daemonJVMOptions memorywiseFitsInto fattestOpts && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) > 0) { + else if (daemonJVMOptions memorywiseFitsInto fattestOpts && FileAgeComparator().compare( + bestDaemonWithMetadata.runFile, + runFile + ) > 0 + ) { // there is at least one bigger, handover my clients to it and shutdown log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE higher prio, handover clients to it and schedule shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") getClients().takeIf { it.isGood }?.let { it.get().forEach { bestDaemonWithMetadata.daemon.registerClient(it) } } scheduleShutdown(true) - } - else { + } else { // undecided, do nothing log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE equal prio, continue: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") // TODO: implement some behaviour here, e.g.: @@ -825,7 +896,7 @@ class CompileServiceImpl( private fun shutdownNow() { log.info("Shutdown started") fun Long.mb() = this / (1024 * 1024) - with (Runtime.getRuntime()) { + with(Runtime.getRuntime()) { log.info("Memory stats: total: ${totalMemory().mb()}mb, free: ${freeMemory().mb()}mb, max: ${maxMemory().mb()}mb") } state.alive.set(Aliveness.Dying.ordinal) @@ -846,14 +917,13 @@ class CompileServiceImpl( state.delayedShutdownQueued.set(false) if (currentClientsCount == state.clientsCounter && currentCompilationsCount == compilationsCounter.get() && - currentSessionId == state.sessions.lastSessionId) - { + currentSessionId == state.sessions.lastSessionId + ) { ifAliveExclusiveUnit(minAliveness = Aliveness.LastSession) { log.fine("Execute delayed shutdown") shutdownNow() } - } - else { + } else { log.info("Cancel delayed shutdown due to a new activity") } } @@ -864,7 +934,8 @@ class CompileServiceImpl( fun shutdownIfIdle() = when { state.sessions.isEmpty() -> shutdownWithDelay() else -> { - daemonOptions.autoshutdownIdleSeconds = TimeUnit.MILLISECONDS.toSeconds(daemonOptions.forceShutdownTimeoutMilliseconds).toInt() + daemonOptions.autoshutdownIdleSeconds = + TimeUnit.MILLISECONDS.toSeconds(daemonOptions.forceShutdownTimeoutMilliseconds).toInt() daemonOptions.autoshutdownUnusedSeconds = daemonOptions.autoshutdownIdleSeconds log.info("Some sessions are active, waiting for them to finish") log.info("Unused/idle timeouts are set to ${daemonOptions.autoshutdownUnusedSeconds}/${daemonOptions.autoshutdownIdleSeconds}s") @@ -879,8 +950,7 @@ class CompileServiceImpl( if (!onAnotherThread) { shutdownIfIdle() - } - else { + } else { timer.schedule(1) { ifAliveExclusiveUnit(minAliveness = Aliveness.LastSession) { shutdownIfIdle() @@ -891,64 +961,79 @@ class CompileServiceImpl( } // todo: remove after remoteIncrementalCompile is removed - private fun doCompile(sessionId: Int, - args: Array, - compilerMessagesStreamProxy: RemoteOutputStream, - serviceOutputStreamProxy: RemoteOutputStream, - operationsTracer: RemoteOperationsTracer?, - body: (PrintStream, EventManager, Profiler) -> ExitCode): CompileService.CallResult = - ifAlive { - withValidClientOrSessionProxy(sessionId) { - operationsTracer?.before("compile") - val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() - val eventManger = EventManagerImpl() - val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerMessagesStreamProxy, rpcProfiler), REMOTE_STREAM_BUFFER_SIZE)) - val serviceOutputStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(serviceOutputStreamProxy, rpcProfiler), REMOTE_STREAM_BUFFER_SIZE)) - try { - val compileServiceReporter = DaemonMessageReporterPrintStreamAdapter(serviceOutputStream) - if (args.none()) - throw IllegalArgumentException("Error: empty arguments list.") - log.info("Starting compilation with args: " + args.joinToString(" ")) - val exitCode = checkedCompile(compileServiceReporter, rpcProfiler) { - body(compilerMessagesStream, eventManger, rpcProfiler).code - } - CompileService.CallResult.Good(exitCode) - } - finally { - serviceOutputStream.flush() - compilerMessagesStream.flush() - eventManger.fireCompilationFinished() - operationsTracer?.after("compile") + private fun doCompile( + sessionId: Int, + args: Array, + compilerMessagesStreamProxy: RemoteOutputStream, + serviceOutputStreamProxy: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer?, + body: (PrintStream, EventManager, Profiler) -> ExitCode + ): CompileService.CallResult = + ifAlive { + withValidClientOrSessionProxy(sessionId) { + operationsTracer?.before("compile") + val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() + val eventManger = EventManagerImpl() + val compilerMessagesStream = PrintStream( + BufferedOutputStream( + RemoteOutputStreamClient(compilerMessagesStreamProxy, rpcProfiler), + REMOTE_STREAM_BUFFER_SIZE + ) + ) + val serviceOutputStream = PrintStream( + BufferedOutputStream( + RemoteOutputStreamClient(serviceOutputStreamProxy, rpcProfiler), + REMOTE_STREAM_BUFFER_SIZE + ) + ) + try { + val compileServiceReporter = DaemonMessageReporterPrintStreamAdapter(serviceOutputStream) + if (args.none()) + throw IllegalArgumentException("Error: empty arguments list.") + log.info("Starting compilation with args: " + args.joinToString(" ")) + val exitCode = checkedCompile(compileServiceReporter, rpcProfiler) { + body(compilerMessagesStream, eventManger, rpcProfiler).code } + CompileService.CallResult.Good(exitCode) + } finally { + serviceOutputStream.flush() + compilerMessagesStream.flush() + eventManger.fireCompilationFinished() + operationsTracer?.after("compile") } } + } - private fun doCompile(sessionId: Int, - daemonMessageReporter: DaemonMessageReporter, - tracer: RemoteOperationsTracer?, - body: (EventManager, Profiler) -> ExitCode): CompileService.CallResult = - ifAlive { - withValidClientOrSessionProxy(sessionId) { - tracer?.before("compile") - val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() - val eventManger = EventManagerImpl() - try { - val exitCode = checkedCompile(daemonMessageReporter, rpcProfiler) { - body(eventManger, rpcProfiler).code - } - CompileService.CallResult.Good(exitCode) - } - finally { - eventManger.fireCompilationFinished() - tracer?.after("compile") + private fun doCompile( + sessionId: Int, + daemonMessageReporter: DaemonMessageReporter, + tracer: RemoteOperationsTracer?, + body: (EventManager, Profiler) -> ExitCode + ): CompileService.CallResult = + ifAlive { + withValidClientOrSessionProxy(sessionId) { + tracer?.before("compile") + val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() + val eventManger = EventManagerImpl() + try { + val exitCode = checkedCompile(daemonMessageReporter, rpcProfiler) { + body(eventManger, rpcProfiler).code } + CompileService.CallResult.Good(exitCode) + } finally { + eventManger.fireCompilationFinished() + tracer?.after("compile") } } + } private fun createCompileServices(facade: CompilerCallbackServicesFacade, eventManager: EventManager, rpcProfiler: Profiler): Services { val builder = Services.Builder() if (facade.hasIncrementalCaches()) { - builder.register(IncrementalCompilationComponents::class.java, RemoteIncrementalCompilationComponentsClient(facade, eventManager, rpcProfiler)) + builder.register( + IncrementalCompilationComponents::class.java, + RemoteIncrementalCompilationComponentsClient(facade, eventManager, rpcProfiler) + ) } if (facade.hasLookupTracker()) { builder.register(LookupTracker::class.java, RemoteLookupTrackerClient(facade, eventManager, rpcProfiler)) @@ -970,7 +1055,7 @@ class CompileServiceImpl( } - private fun checkedCompile(daemonMessageReporter: DaemonMessageReporter, rpcProfiler: Profiler, body: () -> R): R { + private fun checkedCompile(daemonMessageReporter: DaemonMessageReporter, rpcProfiler: Profiler, body: () -> R): R { try { val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler() @@ -986,7 +1071,9 @@ class CompileServiceImpl( val pc = profiler.getTotalCounters() val rpc = rpcProfiler.getTotalCounters() - "PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${"%+d".format(pc.memory.kb())} kb)".let { + "PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${"%+d".format( + pc.memory.kb() + )} kb)".let { daemonMessageReporter.report(ReportSeverity.INFO, it) log.info(it) } @@ -1009,7 +1096,8 @@ class CompileServiceImpl( if (e.cause != null && e.cause != e) { "\nCaused by: ${e.cause}\n ${e.cause!!.stackTrace.joinToString("\n ")}" } else "" - }") + }" + ) throw e } } @@ -1019,7 +1107,10 @@ class CompileServiceImpl( (KotlinCoreEnvironment.applicationEnvironment?.jarFileSystem as? CoreJarFileSystem)?.clearHandlersCache() } - private inline fun ifAlive(minAliveness: Aliveness = Aliveness.LastSession, body: () -> CompileService.CallResult): CompileService.CallResult = rwlock.read { + private inline fun ifAlive( + minAliveness: Aliveness = Aliveness.LastSession, + body: () -> CompileService.CallResult + ): CompileService.CallResult = rwlock.read { ifAliveChecksImpl(minAliveness, body) } @@ -1030,7 +1121,10 @@ class CompileServiceImpl( } } - private inline fun ifAliveExclusive(minAliveness: Aliveness = Aliveness.LastSession, body: () -> CompileService.CallResult): CompileService.CallResult = rwlock.write { + private inline fun ifAliveExclusive( + minAliveness: Aliveness = Aliveness.LastSession, + body: () -> CompileService.CallResult + ): CompileService.CallResult = rwlock.write { ifAliveChecksImpl(minAliveness, body) } @@ -1041,7 +1135,10 @@ class CompileServiceImpl( } } - private inline fun ifAliveChecksImpl(minAliveness: Aliveness = Aliveness.LastSession, body: () -> CompileService.CallResult): CompileService.CallResult { + private inline fun ifAliveChecksImpl( + minAliveness: Aliveness = Aliveness.LastSession, + body: () -> CompileService.CallResult + ): CompileService.CallResult { val curState = state.alive.get() return when { curState < minAliveness.ordinal -> { @@ -1051,8 +1148,7 @@ class CompileServiceImpl( else -> { try { body() - } - catch (e: Throwable) { + } catch (e: Throwable) { log.log(Level.SEVERE, "Exception", e) CompileService.CallResult.Error(e.message ?: "unknown") } @@ -1060,31 +1156,34 @@ class CompileServiceImpl( } } - private inline fun withValidClientOrSessionProxy(sessionId: Int, - body: (ClientOrSessionProxy?) -> CompileService.CallResult + private inline fun withValidClientOrSessionProxy( + sessionId: Int, + body: (ClientOrSessionProxy?) -> CompileService.CallResult ): CompileService.CallResult { val session: ClientOrSessionProxy? = - if (sessionId == CompileService.NO_SESSION) null - else state.sessions[sessionId] ?: return CompileService.CallResult.Error("Unknown or invalid session $sessionId") + if (sessionId == CompileService.NO_SESSION) null + else state.sessions[sessionId] ?: return CompileService.CallResult.Error("Unknown or invalid session $sessionId") try { compilationsCounter.incrementAndGet() return body(session) - } - finally { + } finally { _lastUsedSeconds = nowSeconds() } } - private inline fun withValidRepl(sessionId: Int, body: KotlinJvmReplService.() -> R): CompileService.CallResult = - withValidClientOrSessionProxy(sessionId) { session -> - (session?.data as? KotlinJvmReplService?)?.let { - CompileService.CallResult.Good(it.body()) - } ?: CompileService.CallResult.Error("Not a REPL session $sessionId") - } + private inline fun withValidRepl(sessionId: Int, body: KotlinJvmReplService.() -> R): CompileService.CallResult = + withValidClientOrSessionProxy(sessionId) { session -> + (session?.data as? KotlinJvmReplService?)?.let { + CompileService.CallResult.Good(it.body()) + } ?: CompileService.CallResult.Error("Not a REPL session $sessionId") + } @JvmName("withValidRepl1") - private inline fun withValidRepl(sessionId: Int, body: KotlinJvmReplService.() -> CompileService.CallResult): CompileService.CallResult = - withValidClientOrSessionProxy(sessionId) { session -> - (session?.data as? KotlinJvmReplService?)?.body() ?: CompileService.CallResult.Error("Not a REPL session $sessionId") - } + private inline fun withValidRepl( + sessionId: Int, + body: KotlinJvmReplService.() -> CompileService.CallResult + ): CompileService.CallResult = + withValidClientOrSessionProxy(sessionId) { session -> + (session?.data as? KotlinJvmReplService?)?.body() ?: CompileService.CallResult.Error("Not a REPL session $sessionId") + } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index c936aae5cf1..11ef456e106 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -28,6 +28,8 @@ import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.parsing.classesFqNames +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager +import org.jetbrains.kotlin.incremental.storage.version.saveIfNeeded import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledStatus import java.io.File @@ -37,11 +39,11 @@ abstract class IncrementalCompilerRunner< Args : CommonCompilerArguments, CacheManager : IncrementalCachesManager<*> >( - workingDir: File, - cacheDirName: String, - protected val cacheVersions: List, - protected val reporter: ICReporter, - private val buildHistoryFile: File, + workingDir: File, + cacheDirName: String, + protected val cachesVersionManagers: List, + protected val reporter: ICReporter, + private val buildHistoryFile: File, private val localStateDirs: Collection = emptyList() ) { @@ -268,7 +270,7 @@ abstract class IncrementalCompilerRunner< processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData) if (exitCode == ExitCode.OK) { - cacheVersions.forEach { it.saveIfNeeded() } + cachesVersionManagers.forEach { it.saveIfNeeded() } } return exitCode diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 50ec20cd20b..f52340db28d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -26,6 +26,7 @@ import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.js.* +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory import java.io.File @@ -38,7 +39,7 @@ fun makeJsIncrementally( reporter: ICReporter = EmptyICReporter ) { val isIncremental = IncrementalCompilation.isEnabledForJs() - val versions = commonCacheVersions(cachesDir, isIncremental) + standaloneCacheVersion(cachesDir, isIncremental) + val versions = commonCacheVersionsManagers(cachesDir, isIncremental) + standaloneCacheVersionManager(cachesDir, isIncremental) val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() } .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() val buildHistoryFile = File(cachesDir, "build-history.bin") @@ -64,16 +65,16 @@ inline fun withJsIC(fn: () -> R): R { } class IncrementalJsCompilerRunner( - workingDir: File, - cacheVersions: List, - reporter: ICReporter, + workingDir: File, + cachesVersionManagers: List, + reporter: ICReporter, buildHistoryFile: File, private val modulesApiHistory: ModulesApiHistory ) : IncrementalCompilerRunner( - workingDir, - "caches-js", - cacheVersions, - reporter, + workingDir, + "caches-js", + cachesVersionManagers, + reporter, buildHistoryFile = buildHistoryFile ) { override fun isICEnabled(): Boolean = diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index ea0863eff97..d5dc22e70a1 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -39,6 +39,7 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import org.jetbrains.kotlin.incremental.util.Either import org.jetbrains.kotlin.load.java.JavaClassesTracker import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader @@ -57,7 +58,7 @@ fun makeIncrementally( reporter: ICReporter = EmptyICReporter ) { val isIncremental = IncrementalCompilation.isEnabledForJvm() - val versions = commonCacheVersions(cachesDir, isIncremental) + standaloneCacheVersion(cachesDir, isIncremental) + val versions = commonCacheVersionsManagers(cachesDir, isIncremental) + standaloneCacheVersionManager(cachesDir, isIncremental) val kotlinExtensions = listOf("kt", "kts") val allExtensions = kotlinExtensions + listOf("java") @@ -99,20 +100,20 @@ inline fun withIC(enabled: Boolean = true, fn: ()->R): R { } class IncrementalJvmCompilerRunner( - workingDir: File, - private val javaSourceRoots: Set, - cacheVersions: List, - reporter: ICReporter, - private val usePreciseJavaTracking: Boolean, - buildHistoryFile: File, - localStateDirs: Collection, - private val modulesApiHistory: ModulesApiHistory + workingDir: File, + private val javaSourceRoots: Set, + cachesVersionManagers: List, + reporter: ICReporter, + private val usePreciseJavaTracking: Boolean, + buildHistoryFile: File, + localStateDirs: Collection, + private val modulesApiHistory: ModulesApiHistory ) : IncrementalCompilerRunner( - workingDir, - "caches-jvm", - cacheVersions, - reporter, - localStateDirs = localStateDirs, + workingDir, + "caches-jvm", + cachesVersionManagers, + reporter, + localStateDirs = localStateDirs, buildHistoryFile = buildHistoryFile ) { override fun isICEnabled(): Boolean = diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt index d6370ca1ab4..f96dd496978 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt @@ -16,21 +16,25 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager +import org.jetbrains.kotlin.incremental.storage.version.localCacheVersionManager +import org.jetbrains.kotlin.incremental.storage.version.lookupsCacheVersionManager import java.io.File internal const val STANDALONE_CACHE_VERSION = 2 internal const val STANDALONE_VERSION_FILE_NAME = "standalone-ic-format-version.txt" -fun standaloneCacheVersion(dataRoot: File, enabled: Boolean): CacheVersion = - customCacheVersion(STANDALONE_CACHE_VERSION, STANDALONE_VERSION_FILE_NAME, dataRoot, enabled) +fun standaloneCacheVersionManager(dataRoot: File, enabled: Boolean): CacheVersionManager = + customCacheVersionManager(STANDALONE_CACHE_VERSION, STANDALONE_VERSION_FILE_NAME, dataRoot, enabled) -fun customCacheVersion(version: Int, fileName: String, dataRoot: File, enabled: Boolean): CacheVersion = - CacheVersion(ownVersion = version, - versionFile = File(dataRoot, fileName), - whenVersionChanged = CacheVersion.Action.REBUILD_ALL_KOTLIN, - whenTurnedOn = CacheVersion.Action.REBUILD_ALL_KOTLIN, - whenTurnedOff = CacheVersion.Action.REBUILD_ALL_KOTLIN, - isEnabled = enabled) +fun customCacheVersionManager(version: Int, fileName: String, dataRoot: File, enabled: Boolean): CacheVersionManager = + CacheVersionManager( + File(dataRoot, fileName), + if (enabled) version else null + ) -fun commonCacheVersions(cachesDir: File, enabled: Boolean): List = - listOf(normalCacheVersion(cachesDir, enabled), dataContainerCacheVersion(cachesDir, enabled)) \ No newline at end of file +fun commonCacheVersionsManagers(cachesDir: File, enabled: Boolean): List = + listOf( + localCacheVersionManager(cachesDir, enabled), + lookupsCacheVersionManager(cachesDir, enabled) + ) \ No newline at end of file diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinCompilerWorkspaceSettings.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinCompilerWorkspaceSettings.kt index 8e43765b0f9..9c8d35fe912 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinCompilerWorkspaceSettings.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinCompilerWorkspaceSettings.kt @@ -29,6 +29,10 @@ import com.intellij.util.xmlb.XmlSerializerUtil ) ) class KotlinCompilerWorkspaceSettings : PersistentStateComponent { + /** + * incrementalCompilationForJvmEnabled + * (name `preciseIncrementalEnabled` is kept for workspace file compatibility) + */ var preciseIncrementalEnabled: Boolean = true var incrementalCompilationForJsEnabled: Boolean = false var enableDaemon: Boolean = true diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractDataContainerVersionChangedTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractDataContainerVersionChangedTest.kt index e924e03e809..32307ce5d98 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractDataContainerVersionChangedTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractDataContainerVersionChangedTest.kt @@ -16,14 +16,13 @@ package org.jetbrains.kotlin.jps.build -import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.kotlin.incremental.testingUtils.BuildLogFinder -import org.jetbrains.kotlin.jps.incremental.CacheVersionProvider +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget abstract class AbstractDataContainerVersionChangedTest : AbstractIncrementalCacheVersionChangedTest() { override val buildLogFinder: BuildLogFinder get() = BuildLogFinder(isDataContainerBuildLogEnabled = true) - override fun getVersions(cacheVersionProvider: CacheVersionProvider, targets: Iterable) = - listOf(cacheVersionProvider.dataContainerVersion()) + override fun getVersionManagersToTest(target: KotlinModuleBuildTarget<*>) = + listOf(kotlinCompileContext.lookupsCacheAttributesManager.versionManagerForTesting) } \ No newline at end of file diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt index 5c3c8996a30..3a32beb9494 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalCacheVersionChangedTest.kt @@ -16,15 +16,14 @@ package org.jetbrains.kotlin.jps.build -import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import org.jetbrains.kotlin.incremental.testingUtils.Modification import org.jetbrains.kotlin.incremental.testingUtils.ModifyContent -import org.jetbrains.kotlin.jps.incremental.CacheVersionProvider +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget abstract class AbstractIncrementalCacheVersionChangedTest : AbstractIncrementalJpsTest(allowNoFilesWithSuffixInTestData = true) { override fun performAdditionalModifications(modifications: List) { val modifiedFiles = modifications.filterIsInstance().map { it.path } - val paths = projectDescriptor.dataManager.dataPaths val targets = projectDescriptor.allModuleTargets val hasKotlin = HasKotlinMarker(projectDescriptor.dataManager) @@ -33,13 +32,18 @@ abstract class AbstractIncrementalCacheVersionChangedTest : AbstractIncrementalJ } if (modifiedFiles.none { it.endsWith("do-not-change-cache-versions") }) { - val cacheVersionProvider = CacheVersionProvider(paths, isIncrementalCompilationEnabled = true) - val versions = getVersions(cacheVersionProvider, targets) - val versionFiles = versions.map { it.formatVersionFile }.filter { it.exists() } - versionFiles.forEach { it.writeText("777") } + val versions = targets.flatMap { + getVersionManagersToTest(kotlinCompileContext.targetsBinding[it]!!) + } + + versions.forEach { + if (it.versionFileForTesting.exists()) { + it.versionFileForTesting.writeText("777") + } + } } } - protected open fun getVersions(cacheVersionProvider: CacheVersionProvider, targets: Iterable) = - targets.map { cacheVersionProvider.normalVersion(it) } + protected open fun getVersionManagersToTest(target: KotlinModuleBuildTarget<*>): List = + listOf(target.localCacheVersionManager) } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt index c906853878a..5b745fb09f0 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt @@ -44,13 +44,15 @@ import org.jetbrains.jps.model.java.JpsJavaDependencyScope import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.util.JpsPathUtil import org.jetbrains.kotlin.config.IncrementalCompilation -import org.jetbrains.kotlin.incremental.CacheVersion import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.isJavaFile +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import org.jetbrains.kotlin.incremental.testingUtils.* +import org.jetbrains.kotlin.jps.incremental.CompositeLookupsCacheAttributesManager import org.jetbrains.kotlin.jps.incremental.getKotlinCache import org.jetbrains.kotlin.jps.incremental.withLookupStorage -import org.jetbrains.kotlin.jps.platforms.kotlinBuildTargets +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.test.KotlinTestUtils import org.jetbrains.kotlin.utils.Printer import org.jetbrains.kotlin.utils.keysToMap @@ -82,6 +84,8 @@ abstract class AbstractIncrementalJpsTest( protected var mapWorkingToOriginalFile: MutableMap = hashMapOf() + lateinit var kotlinCompileContext: KotlinCompileContext + protected open val buildLogFinder: BuildLogFinder get() = BuildLogFinder() @@ -145,19 +149,21 @@ abstract class AbstractIncrementalJpsTest( projectDescriptor = createProjectDescriptor(BuildLoggingManager(logger)) val lookupTracker = TestLookupTracker() - projectDescriptor.project.setTestingContext(TestingContext(lookupTracker, logger)) + val testingContext = TestingContext(lookupTracker, logger) + projectDescriptor.project.setTestingContext(testingContext) try { val builder = IncProjectBuilder(projectDescriptor, BuilderRegistry.getInstance(), myBuildParams, CanceledStatus.NULL, mockConstantSearch, true) val buildResult = BuildResult() builder.addMessageHandler(buildResult) val finalScope = scope.build() + builder.build(finalScope, false) - lookupTracker.lookups.mapTo(lookupsDuringTest) { LookupSymbol(it.name, it.scopeFqName) } + // testingContext.kotlinCompileContext is initialized in KotlinBuilder.initializeKotlinContext + kotlinCompileContext = testingContext.kotlinCompileContext!! - // for getting kotlin platform only - val dummyCompileContext = CompileContextImpl.createContextForTests(finalScope, projectDescriptor) + lookupTracker.lookups.mapTo(lookupsDuringTest) { LookupSymbol(it.name, it.scopeFqName) } if (!buildResult.isSuccessful) { val errorMessages = @@ -169,7 +175,7 @@ abstract class AbstractIncrementalJpsTest( return MakeResult(logger.log + "$COMPILATION_FAILED\n" + errorMessages + "\n", true, null) } else { - return MakeResult(logger.log, false, createMappingsDump(projectDescriptor, dummyCompileContext)) + return MakeResult(logger.log, false, createMappingsDump(projectDescriptor)) } } finally { @@ -287,20 +293,18 @@ abstract class AbstractIncrementalJpsTest( } private fun createMappingsDump( - project: ProjectDescriptor, - dummyCompileContext: CompileContext - ) = createKotlinIncrementalCacheDump(project, dummyCompileContext) + "\n\n\n" + + project: ProjectDescriptor + ) = createKotlinIncrementalCacheDump(project) + "\n\n\n" + createLookupCacheDump(project) + "\n\n\n" + createCommonMappingsDump(project) + "\n\n\n" + createJavaMappingsDump(project) private fun createKotlinIncrementalCacheDump( - project: ProjectDescriptor, - dummyCompileContext: CompileContext + project: ProjectDescriptor ): String { return buildString { for (target in project.allModuleTargets.sortedBy { it.presentableName }) { - val kotlinCache = project.dataManager.getKotlinCache(dummyCompileContext.kotlinBuildTargets[target]) + val kotlinCache = project.dataManager.getKotlinCache(kotlinCompileContext.targetsBinding[target]) if (kotlinCache != null) { append("\n") append(kotlinCache.dump()) @@ -443,14 +447,23 @@ abstract class AbstractIncrementalJpsTest( override fun doGetProjectDir(): File? = workDir - private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase(), BuildLogger { + private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase(), TestingBuildLogger { private val markedDirtyBeforeRound = ArrayList() private val markedDirtyAfterRound = ArrayList() - override fun actionsOnCacheVersionChanged(actions: List) { - if (actions.size > 1 && actions.any { it != CacheVersion.Action.DO_NOTHING }) { - logLine("Actions after cache changed: $actions") + override fun invalidOrUnusedCache( + chunk: KotlinChunk?, + target: KotlinModuleBuildTarget<*>?, + attributesDiff: CacheAttributesDiff<*> + ) { + val cacheManager = attributesDiff.manager + val cacheTitle = when (cacheManager) { + is CacheVersionManager -> "Local cache for ${chunk ?: target}" + is CompositeLookupsCacheAttributesManager -> "Lookups cache" + else -> error("Unknown cache manager $cacheManager") } + + logLine("$cacheTitle are $attributesDiff") } override fun markedAsDirtyBeforeRound(files: Iterable) { @@ -461,13 +474,15 @@ abstract class AbstractIncrementalJpsTest( markedDirtyAfterRound.addAll(files) } - override fun buildStarted(context: CompileContext, chunk: ModuleChunk) { + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + logDirtyFiles(markedDirtyBeforeRound) // files can be marked as dirty during build start (KotlinCompileContext initialization) + if (!chunk.isDummy(context) && context.projectDescriptor.project.modules.size > 1) { logLine("Building ${chunk.modules.sortedBy { it.name }.joinToString { it.name }}") } } - override fun afterBuildStarted(context: CompileContext, chunk: ModuleChunk) { + override fun afterChunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { logDirtyFiles(markedDirtyBeforeRound) } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.173 b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.173 index b12ab12dab7..47b718e7b29 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.173 +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.173 @@ -44,13 +44,14 @@ import org.jetbrains.jps.model.java.JpsJavaDependencyScope import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.util.JpsPathUtil import org.jetbrains.kotlin.config.IncrementalCompilation -import org.jetbrains.kotlin.incremental.CacheVersion import org.jetbrains.kotlin.incremental.LookupSymbol -import org.jetbrains.kotlin.incremental.isJavaFile +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import org.jetbrains.kotlin.incremental.testingUtils.* +import org.jetbrains.kotlin.jps.incremental.CompositeLookupsCacheAttributesManager import org.jetbrains.kotlin.jps.incremental.getKotlinCache import org.jetbrains.kotlin.jps.incremental.withLookupStorage -import org.jetbrains.kotlin.jps.platforms.kotlinBuildTargets +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.test.KotlinTestUtils import org.jetbrains.kotlin.utils.Printer import org.jetbrains.kotlin.utils.keysToMap @@ -82,6 +83,8 @@ abstract class AbstractIncrementalJpsTest( protected var mapWorkingToOriginalFile: MutableMap = hashMapOf() + lateinit var kotlinCompileContext: KotlinCompileContext + protected open val buildLogFinder: BuildLogFinder get() = BuildLogFinder() @@ -144,19 +147,20 @@ abstract class AbstractIncrementalJpsTest( projectDescriptor = createProjectDescriptor(BuildLoggingManager(logger)) val lookupTracker = TestLookupTracker() - projectDescriptor.project.setTestingContext(TestingContext(lookupTracker, logger)) + val testingContext = TestingContext(lookupTracker, logger) + projectDescriptor.project.setTestingContext(testingContext) try { val builder = IncProjectBuilder(projectDescriptor, BuilderRegistry.getInstance(), myBuildParams, CanceledStatus.NULL, mockConstantSearch, true) val buildResult = BuildResult() builder.addMessageHandler(buildResult) val finalScope = scope.build() + builder.build(finalScope, false) - lookupTracker.lookups.mapTo(lookupsDuringTest) { LookupSymbol(it.name, it.scopeFqName) } + // testingContext.kotlinCompileContext is initialized in KotlinBuilder.initializeKotlinContext + kotlinCompileContext = testingContext.kotlinCompileContext!! - // for getting kotlin platform only - val dummyCompileContext = CompileContextImpl.createContextForTests(finalScope, projectDescriptor) if (!buildResult.isSuccessful) { val errorMessages = @@ -168,7 +172,7 @@ abstract class AbstractIncrementalJpsTest( return MakeResult(logger.log + "$COMPILATION_FAILED\n" + errorMessages + "\n", true, null) } else { - return MakeResult(logger.log, false, createMappingsDump(projectDescriptor, dummyCompileContext)) + return MakeResult(logger.log, false, createMappingsDump(projectDescriptor)) } } finally { @@ -286,20 +290,18 @@ abstract class AbstractIncrementalJpsTest( } private fun createMappingsDump( - project: ProjectDescriptor, - dummyCompileContext: CompileContext - ) = createKotlinIncrementalCacheDump(project, dummyCompileContext) + "\n\n\n" + + project: ProjectDescriptor + ) = createKotlinIncrementalCacheDump(project) + "\n\n\n" + createLookupCacheDump(project) + "\n\n\n" + createCommonMappingsDump(project) + "\n\n\n" + createJavaMappingsDump(project) private fun createKotlinIncrementalCacheDump( - project: ProjectDescriptor, - dummyCompileContext: CompileContext + project: ProjectDescriptor ): String { return buildString { for (target in project.allModuleTargets.sortedBy { it.presentableName }) { - val kotlinCache = project.dataManager.getKotlinCache(dummyCompileContext.kotlinBuildTargets[target]) + val kotlinCache = project.dataManager.getKotlinCache(kotlinCompileContext.targetsBinding[target]) if (kotlinCache != null) { append("\n") append(kotlinCache.dump()) @@ -442,14 +444,23 @@ abstract class AbstractIncrementalJpsTest( override fun doGetProjectDir(): File? = workDir - private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase(), BuildLogger { + private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase(), TestingBuildLogger { private val markedDirtyBeforeRound = ArrayList() private val markedDirtyAfterRound = ArrayList() - override fun actionsOnCacheVersionChanged(actions: List) { - if (actions.size > 1 && actions.any { it != CacheVersion.Action.DO_NOTHING }) { - logLine("Actions after cache changed: $actions") + override fun invalidOrUnusedCache( + chunk: KotlinChunk?, + target: KotlinModuleBuildTarget<*>?, + attributesDiff: CacheAttributesDiff<*> + ) { + val cacheManager = attributesDiff.manager + val cacheTitle = when (cacheManager) { + is CacheVersionManager -> "Local cache for ${chunk ?: target}" + is CompositeLookupsCacheAttributesManager -> "Lookups cache" + else -> error("Unknown cache manager $cacheManager") } + + logLine("$cacheTitle are $attributesDiff") } override fun markedAsDirtyBeforeRound(files: Iterable) { @@ -460,13 +471,15 @@ abstract class AbstractIncrementalJpsTest( markedDirtyAfterRound.addAll(files) } - override fun buildStarted(context: CompileContext, chunk: ModuleChunk) { + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + logDirtyFiles(markedDirtyBeforeRound) // files can be marked as dirty during build start (KotlinCompileContext initialization) + if (!chunk.isDummy(context) && context.projectDescriptor.project.modules.size > 1) { logLine("Building ${chunk.modules.sortedBy { it.name }.joinToString { it.name }}") } } - override fun afterBuildStarted(context: CompileContext, chunk: ModuleChunk) { + override fun afterChunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { logDirtyFiles(markedDirtyBeforeRound) } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalLazyCachesTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalLazyCachesTest.kt index 97fc37f55d5..cc530f70284 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalLazyCachesTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalLazyCachesTest.kt @@ -18,17 +18,14 @@ package org.jetbrains.kotlin.jps.build import com.intellij.testFramework.UsefulTestCase import org.jetbrains.jps.builders.BuildTarget -import org.jetbrains.jps.builders.CompileScopeTestBuilder import org.jetbrains.jps.builders.storage.BuildDataPaths -import org.jetbrains.jps.incremental.CompileContextImpl import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.incremental.KOTLIN_CACHE_DIRECTORY_NAME import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import org.jetbrains.kotlin.incremental.testingUtils.Modification import org.jetbrains.kotlin.incremental.testingUtils.ModifyContent -import org.jetbrains.kotlin.jps.incremental.CacheVersionProvider import org.jetbrains.kotlin.jps.incremental.KotlinDataContainerTarget -import org.jetbrains.kotlin.jps.platforms.kotlinBuildTargets +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.utils.Printer import java.io.File @@ -85,44 +82,55 @@ abstract class AbstractIncrementalLazyCachesTest : AbstractIncrementalJpsTest() private fun dumpKotlinCachesFileNames(): String { val sb = StringBuilder() - val p = Printer(sb) - val targets = projectDescriptor.allModuleTargets + val printer = Printer(sb) + val chunks = kotlinCompileContext.targetsIndex.chunks val dataManager = projectDescriptor.dataManager val paths = dataManager.dataPaths - val versions = CacheVersionProvider(paths, isIncrementalCompilationEnabled = true) - dumpCachesForTarget(p, paths, KotlinDataContainerTarget, versions.dataContainerVersion().formatVersionFile) + dumpCachesForTarget( + printer, + paths, + KotlinDataContainerTarget, + kotlinCompileContext.lookupsCacheAttributesManager.versionManagerForTesting.versionFileForTesting + ) - // for getting kotlin platform only - val dummyCompileContext = CompileContextImpl.createContextForTests(CompileScopeTestBuilder.make().build(), projectDescriptor) + data class TargetInChunk(val chunk: KotlinChunk, val target: KotlinModuleBuildTarget<*>) - for (target in targets.sortedBy { it.presentableName }) { - val kotlinModuleBuildTarget = dummyCompileContext.kotlinBuildTargets[target]!! - val metaBuildInfo = kotlinModuleBuildTarget.buildMetaInfoFile(target, dataManager) - dumpCachesForTarget(p, paths, target, - versions.normalVersion(target).formatVersionFile, - metaBuildInfo, - subdirectory = KOTLIN_CACHE_DIRECTORY_NAME) + val allTargets = chunks.flatMap { chunk -> + chunk.targets.map { target -> + TargetInChunk(chunk, target) + } + }.sortedBy { it.target.jpsModuleBuildTarget.presentableName } + + allTargets.forEach { (chunk, target) -> + val metaBuildInfo = chunk.buildMetaInfoFile(target.jpsModuleBuildTarget) + dumpCachesForTarget( + printer, paths, target.jpsModuleBuildTarget, + target.localCacheVersionManager.versionFileForTesting, + metaBuildInfo, + subdirectory = KOTLIN_CACHE_DIRECTORY_NAME + ) } + return sb.toString() } private fun dumpCachesForTarget( - p: Printer, - paths: BuildDataPaths, - target: BuildTarget<*>, - vararg cacheVersionsFiles: File, - subdirectory: String? = null + p: Printer, + paths: BuildDataPaths, + target: BuildTarget<*>, + vararg cacheVersionsFiles: File, + subdirectory: String? = null ) { p.println(target) p.pushIndent() val dataRoot = paths.getTargetDataRoot(target).let { if (subdirectory != null) File(it, subdirectory) else it } cacheVersionsFiles - .filter(File::exists) - .sortedBy { it.name } - .forEach { p.println(it.name) } + .filter(File::exists) + .sortedBy { it.name } + .forEach { p.println(it.name) } kotlinCacheNames(dataRoot).sorted().forEach { p.println(it) } diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt index b1535fd080d..f59bb906dd4 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt @@ -58,13 +58,13 @@ import org.jetbrains.kotlin.codegen.AsmUtil import org.jetbrains.kotlin.codegen.JvmCodegenUtil import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.config.KotlinCompilerVersion.TEST_IS_PRE_RELEASE_SYSTEM_PROPERTY -import org.jetbrains.kotlin.incremental.CacheVersion import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff import org.jetbrains.kotlin.incremental.withIC import org.jetbrains.kotlin.jps.build.KotlinJpsBuildTest.LibraryDependency.* import org.jetbrains.kotlin.jps.model.kotlinCommonCompilerArguments import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments -import org.jetbrains.kotlin.jps.platforms.productionBuildTarget +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.test.KotlinTestUtils @@ -970,6 +970,9 @@ open class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() { } } + // c -> b -exported-> a + // c2 -> b2 ------------^ + val a = addModuleWithSourceAndTestRoot("a") val b = addModuleWithSourceAndTestRoot("b") val c = addModuleWithSourceAndTestRoot("c") @@ -983,15 +986,21 @@ open class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() { val actual = StringBuilder() buildCustom(CanceledStatus.NULL, TestProjectBuilderLogger(), BuildResult()) { - project.setTestingContext(TestingContext(LookupTracker.DO_NOTHING, object : BuildLogger { - override fun buildStarted(context: CompileContext, chunk: ModuleChunk) { - actual.append("Targets dependent on ${chunk.targets.joinToString() }:\n") - actual.append(getDependentTargets(chunk.targets, context).map { it.toString() }.sorted().joinToString("\n")) + project.setTestingContext(TestingContext(LookupTracker.DO_NOTHING, object : TestingBuildLogger { + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + actual.append("Targets dependent on ${chunk.targets.joinToString()}:\n") + val dependentRecursively = mutableSetOf() + context.kotlin.getChunk(chunk)!!.collectDependentChunksRecursivelyExportedOnly(dependentRecursively) + dependentRecursively.asSequence().map { it.targets.joinToString() }.sorted().joinTo(actual, "\n") actual.append("\n---------\n") } - override fun afterBuildStarted(context: CompileContext, chunk: ModuleChunk) {} - override fun actionsOnCacheVersionChanged(actions: List) {} + override fun afterChunkBuildStarted(context: CompileContext, chunk: ModuleChunk) {} + override fun invalidOrUnusedCache( + chunk: KotlinChunk?, + target: KotlinModuleBuildTarget<*>?, + attributesDiff: CacheAttributesDiff<*> + ) {} override fun buildFinished(exitCode: ModuleLevelBuilder.ExitCode) {} override fun markedAsDirtyBeforeRound(files: Iterable) {} override fun markedAsDirtyAfterRound(files: Iterable) {} diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributesManagerTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributesManagerTest.kt new file mode 100644 index 00000000000..926105dca9c --- /dev/null +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributesManagerTest.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.incremental + +import org.jetbrains.kotlin.incremental.storage.version.CacheStatus +import org.jetbrains.kotlin.incremental.storage.version.loadDiff +import org.junit.Test +import java.io.File +import kotlin.test.assertEquals + +class CompositeLookupsCacheAttributesManagerTest { + val manager = CompositeLookupsCacheAttributesManager(File("not-used"), setOf()) + + @Test + fun testNothingToJava() { + assertEquals( + CacheStatus.INVALID, + manager.loadDiff( + actual = null, + expected = CompositeLookupsCacheAttributes(1, setOf("jvm")) + ).status + ) + } + + @Test + fun testNothingToJavaAndJs() { + assertEquals( + CacheStatus.INVALID, + manager.loadDiff( + actual = null, + expected = CompositeLookupsCacheAttributes(1, setOf("jvm", "js")) + ).status + ) + } + + @Test + fun testJsToJava() { + assertEquals( + CacheStatus.INVALID, + manager.loadDiff( + actual = CompositeLookupsCacheAttributes(1, setOf("jvm")), + expected = CompositeLookupsCacheAttributes(1, setOf("js")) + ).status + ) + } + + @Test + fun testJsAndJavaToJava() { + assertEquals( + CacheStatus.VALID, + manager.loadDiff( + actual = CompositeLookupsCacheAttributes(1, setOf("jvm", "js")), + expected = CompositeLookupsCacheAttributes(1, setOf("jvm")) + ).status + ) + } + + @Test + fun testJsAndJavaToJavaWithOtherVersion() { + assertEquals( + CacheStatus.INVALID, + manager.loadDiff( + actual = CompositeLookupsCacheAttributes(1, setOf("jvm", "js")), + expected = CompositeLookupsCacheAttributes(2, setOf("jvm")) + ).status + ) + } + + @Test + fun testJavaToJsAndJava() { + assertEquals( + CacheStatus.INVALID, + manager.loadDiff( + actual = CompositeLookupsCacheAttributes(1, setOf("jvm")), + expected = CompositeLookupsCacheAttributes(1, setOf("jvm", "js")) + ).status + ) + } +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/FSOperationsHelper.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/FSOperationsHelper.kt index cfdee997b26..590ec801d3b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/FSOperationsHelper.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/FSOperationsHelper.kt @@ -89,7 +89,9 @@ class FSOperationsHelper( } fun markInChunkOrDependents(files: Iterable, excludeFiles: Set) { - markFilesImpl(files, currentRound = false) { it !in excludeFiles && it.exists() && moduleBasedFilter.accept(it) } + markFilesImpl(files, currentRound = false) { + it !in excludeFiles && it.exists() && moduleBasedFilter.accept(it) + } } private inline fun markFilesImpl( diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 761a4c7c543..f4fb63bc15b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -17,9 +17,7 @@ package org.jetbrains.kotlin.jps.build import com.intellij.openapi.diagnostic.Logger -import com.intellij.util.containers.ContainerUtil import org.jetbrains.jps.ModuleChunk -import org.jetbrains.jps.builders.BuildTarget import org.jetbrains.jps.builders.DirtyFilesHolder import org.jetbrains.jps.builders.FileProcessor import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase @@ -30,9 +28,6 @@ import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.* import org.jetbrains.jps.incremental.java.JavaBuilder import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.model.JpsProject -import org.jetbrains.jps.model.java.JpsJavaClasspathKind -import org.jetbrains.jps.model.java.JpsJavaExtensionService -import org.jetbrains.jps.model.module.JpsModule import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity @@ -45,20 +40,20 @@ import org.jetbrains.kotlin.daemon.common.isDaemonEnabled import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.jps.incremental.* +import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache +import org.jetbrains.kotlin.jps.incremental.withLookupStorage import org.jetbrains.kotlin.jps.model.kotlinKind -import org.jetbrains.kotlin.jps.platforms.KotlinJvmModuleBuildTarget -import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget -import org.jetbrains.kotlin.jps.platforms.kotlinBuildTargets +import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.preloading.ClassCondition import org.jetbrains.kotlin.utils.KotlinPaths import org.jetbrains.kotlin.utils.KotlinPathsFromHomeDir import org.jetbrains.kotlin.utils.PathUtil -import org.jetbrains.kotlin.utils.keysToMapExceptNulls import java.io.File import java.util.* import kotlin.collections.HashSet +import kotlin.system.measureTimeMillis class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { companion object { @@ -89,12 +84,16 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { override fun getCompilableFileExtensions() = arrayListOf("kt", "kts") override fun buildStarted(context: CompileContext) { + logSettings(context) + } + + private fun logSettings(context: CompileContext) { LOG.debug("==========================================") LOG.info("is Kotlin incremental compilation enabled for JVM: ${IncrementalCompilation.isEnabledForJvm()}") LOG.info("is Kotlin incremental compilation enabled for JS: ${IncrementalCompilation.isEnabledForJs()}") if (IncrementalCompilation.isEnabledForJs()) { val messageCollector = MessageCollectorAdapter(context, null) - messageCollector.report(CompilerMessageSeverity.INFO, "Using experimental incremental compilation for Kotlin/JS") + messageCollector.report(INFO, "Using experimental incremental compilation for Kotlin/JS") } LOG.info("is Kotlin compiler daemon enabled: ${isDaemonEnabled()}") @@ -105,8 +104,66 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { } } - override fun buildFinished(context: CompileContext?) { - statisticsLogger.reportTotal() + /** + * Ensure Kotlin Context initialized. + * Kotlin Context should be initialized only when required (before first kotlin chunk build). + */ + private fun ensureKotlinContextInitialized(context: CompileContext): KotlinCompileContext { + val kotlinCompileContext = context.getUserData(kotlinCompileContextKey) + if (kotlinCompileContext != null) return kotlinCompileContext + + // don't synchronize on context, since it is chunk local only + synchronized(kotlinCompileContextKey) { + val actualKotlinCompileContext = context.getUserData(kotlinCompileContextKey) + if (actualKotlinCompileContext != null) return actualKotlinCompileContext + + try { + return initializeKotlinContext(context) + } catch (t: Throwable) { + jpsReportInternalBuilderError(context, Error("Cannot initialize Kotlin context: ${t.message}", t)) + throw t + } + } + } + + private fun initializeKotlinContext(context: CompileContext): KotlinCompileContext { + lateinit var kotlinContext: KotlinCompileContext + + val time = measureTimeMillis { + kotlinContext = KotlinCompileContext(context) + + context.putUserData(kotlinCompileContextKey, kotlinContext) + context.testingContext?.kotlinCompileContext = kotlinContext + + if (kotlinContext.shouldCheckCacheVersions && kotlinContext.hasKotlin()) { + kotlinContext.checkCacheVersions() + } + + kotlinContext.cleanupCaches() + } + + LOG.info("Total Kotlin global compile context initialization time: $time ms") + + return kotlinContext + } + + override fun buildFinished(context: CompileContext) { + ensureKotlinContextDisposed(context) + } + + private fun ensureKotlinContextDisposed(context: CompileContext) { + if (context.getUserData(kotlinCompileContextKey) != null) { + // don't synchronize on context, since it chunk local only + synchronized(kotlinCompileContextKey) { + val kotlinCompileContext = context.getUserData(kotlinCompileContextKey) + if (kotlinCompileContext != null) { + kotlinCompileContext.dispose() + context.putUserData(kotlinCompileContextKey, null) + + statisticsLogger.reportTotal() + } + } + } } override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { @@ -114,18 +171,40 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { if (chunk.isDummy(context)) return + val kotlinContext = ensureKotlinContextInitialized(context) + val buildLogger = context.testingContext?.buildLogger - buildLogger?.buildStarted(context, chunk) + buildLogger?.chunkBuildStarted(context, chunk) if (JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) return val targets = chunk.targets - val dataManager = context.projectDescriptor.dataManager - val hasKotlin = HasKotlinMarker(dataManager) + if (targets.none { kotlinContext.hasKotlinMarker[it] == true }) return - if (targets.none { hasKotlin[it] == true }) return + val kotlinChunk = kotlinContext.getChunk(chunk) ?: return + kotlinContext.checkChunkCacheVersion(kotlinChunk) - val roundDirtyFiles = KotlinDirtySourceFilesHolder( + if (!kotlinContext.rebuildingAllKotlin) { + markAdditionalFilesForInitialRound(kotlinChunk, chunk, kotlinContext) + } + + buildLogger?.afterChunkBuildStarted(context, chunk) + } + + /** + * Invalidate usages of removed classes. + * See KT-13677 for more details. + * + * todo(1.2.80): move to KotlinChunk + * todo(1.2.80): got rid of jpsGlobalContext usages (replace with KotlinCompileContext) + */ + private fun markAdditionalFilesForInitialRound( + kotlinChunk: KotlinChunk, + chunk: ModuleChunk, + kotlinContext: KotlinCompileContext + ) { + val context = kotlinContext.jpsContext + val dirtyFilesHolder = KotlinDirtySourceFilesHolder( chunk, context, object : DirtyFilesHolderBase(context) { @@ -134,50 +213,16 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { } } ) - val fsOperations = FSOperationsHelper(context, chunk, roundDirtyFiles, LOG) + val fsOperations = FSOperationsHelper(context, chunk, dirtyFilesHolder, LOG) - val jpsRepresentativeTarget = chunk.representativeTarget() - val representativeTarget = context.kotlinBuildTargets[jpsRepresentativeTarget] - if (representativeTarget == null) { - LOG.warn("Unable to find Kotlin build target for JPS target ${jpsRepresentativeTarget.presentableName}") - } - if (System.getProperty(SKIP_CACHE_VERSION_CHECK_PROPERTY) == null && representativeTarget != null) { - val cacheVersionsProvider = CacheVersionProvider(dataManager.dataPaths, representativeTarget.isIncrementalCompilationEnabled) - val actions = checkCachesVersions(context, cacheVersionsProvider, chunk) - applyActionsOnCacheVersionChange(actions, cacheVersionsProvider, context, dataManager, targets, fsOperations) - if (CacheVersion.Action.REBUILD_ALL_KOTLIN in actions) { - return - } - } + val representativeTarget = kotlinContext.targetsBinding[chunk.representativeTarget()] ?: return - // try to perform a lookup - // request rebuild if storage is corrupted - try { - dataManager.withLookupStorage { - it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>")) - } - } catch (e: Exception) { - // todo: report to Intellij when IDEA-187115 is implemented - LOG.info(e) - markAllKotlinForRebuild(context, fsOperations, "Lookup storage is corrupted") - return - } + // dependent caches are not required, since we are not going to update caches + val incrementalCaches = kotlinChunk.loadCaches(loadDependent = false) - markAdditionalFilesForInitialRound(chunk, context, fsOperations, roundDirtyFiles) - buildLogger?.afterBuildStarted(context, chunk) - } - - private fun markAdditionalFilesForInitialRound( - chunk: ModuleChunk, - context: CompileContext, - fsOperations: FSOperationsHelper, - dirtyFilesHolder: KotlinDirtySourceFilesHolder - ) { - val representativeTarget = context.kotlinBuildTargets[chunk.representativeTarget()] ?: return - - val incrementalCaches = getIncrementalCaches(chunk, context) val messageCollector = MessageCollectorAdapter(context, representativeTarget) val environment = createCompileEnvironment( + kotlinContext.jpsContext, representativeTarget, incrementalCaches, LookupTracker.DO_NOTHING, @@ -187,10 +232,10 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { ) ?: return val removedClasses = HashSet() - for (target in chunk.targets) { + for (target in kotlinChunk.targets) { val cache = incrementalCaches[target] ?: continue - val dirtyFiles = dirtyFilesHolder.getDirtyFiles(target) - val removedFiles = dirtyFilesHolder.getRemovedFiles(target) + val dirtyFiles = dirtyFilesHolder.getDirtyFiles(target.jpsModuleBuildTarget) + val removedFiles = dirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget) val existingClasses = JpsKotlinCompilerRunner().classesFqNamesByFiles(environment, dirtyFiles) val previousClasses = cache.classesFqNamesBySources(dirtyFiles + removedFiles) @@ -209,23 +254,6 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { fsOperations.markFilesForCurrentRound(affectedByRemovedClasses) } - private fun checkCachesVersions( - context: CompileContext, - cacheVersionsProvider: CacheVersionProvider, - chunk: ModuleChunk - ): Set { - val targets = chunk.targets - val dataManager = context.projectDescriptor.dataManager - - val allVersions = cacheVersionsProvider.allVersions(targets) - val actions = allVersions.map { it.checkVersion() }.toMutableSet() - - val kotlinModuleBuilderTarget = context.kotlinBuildTargets[chunk.representativeTarget()]!! - kotlinModuleBuilderTarget.checkCachesVersions(chunk, dataManager, actions) - - return actions - } - override fun chunkBuildFinished(context: CompileContext, chunk: ModuleChunk) { super.chunkBuildFinished(context, chunk) @@ -252,7 +280,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { if (chunk.modules.size > 1) { messageCollector.report( CompilerMessageSeverity.ERROR, - "Cyclically dependent modules is not supported in multiplatform projects" + "Cyclically dependent modules are not supported in multiplatform projects" ) return ABORT } @@ -299,39 +327,47 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { return NOTHING_DONE } + val kotlinContext = context.kotlin val projectDescriptor = context.projectDescriptor val dataManager = projectDescriptor.dataManager val targets = chunk.targets - val hasKotlin = HasKotlinMarker(dataManager) - val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context) - || targets.any { rebuildAfterCacheVersionChanged[it] == true } + || targets.any { kotlinContext.rebuildAfterCacheVersionChanged[it] == true } - if (kotlinDirtyFilesHolder.hasDirtyOrRemovedFiles) { - if (!isChunkRebuilding && !representativeTarget.isIncrementalCompilationEnabled) { - targets.forEach { rebuildAfterCacheVersionChanged[it] = true } - return CHUNK_REBUILD_REQUIRED - } - } else { + if (!kotlinDirtyFilesHolder.hasDirtyOrRemovedFiles) { if (isChunkRebuilding) { - targets.forEach { hasKotlin[it] = false } + targets.forEach { + kotlinContext.hasKotlinMarker[it] = false + } } - targets.forEach { rebuildAfterCacheVersionChanged.clean(it) } + targets.forEach { kotlinContext.rebuildAfterCacheVersionChanged.clean(it) } return NOTHING_DONE } + // Request CHUNK_REBUILD when IC is off and there are dirty Kotlin files + // Otherwise unexpected compile error might happen, when there are Groovy files, + // but they are not dirty, so Groovy builder does not generate source stubs, + // and Kotlin builder is filtering out output directory from classpath + // (because it may contain outdated Java classes). + if (!isChunkRebuilding && !representativeTarget.isIncrementalCompilationEnabled) { + targets.forEach { kotlinContext.rebuildAfterCacheVersionChanged[it] = true } + return CHUNK_REBUILD_REQUIRED + } + val targetsWithoutOutputDir = targets.filter { it.outputDir == null } if (targetsWithoutOutputDir.isNotEmpty()) { messageCollector.report(ERROR, "Output directory not specified for " + targetsWithoutOutputDir.joinToString()) return ABORT } + val kotlinChunk = chunk.toKotlinChunk(context)!! val project = projectDescriptor.project val lookupTracker = getLookupTracker(project, representativeTarget) val exceptActualTracer = ExpectActualTrackerImpl() - val incrementalCaches = getIncrementalCaches(chunk, context) + val incrementalCaches = kotlinChunk.loadCaches() val environment = createCompileEnvironment( + context, representativeTarget, incrementalCaches, lookupTracker, @@ -340,7 +376,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { messageCollector ) ?: return ABORT - val commonArguments = representativeTarget.compilerArgumentsForChunk(chunk).apply { + val commonArguments = kotlinChunk.compilerArguments.apply { reportOutputFiles = true version = true // Always report the version to help diagnosing user issues if they submit the compiler output if (languageVersion == null) languageVersion = VersionView.RELEASED_VERSION.versionString @@ -352,8 +388,15 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val start = System.nanoTime() val outputItemCollector = doCompileModuleChunk( - chunk, representativeTarget, commonArguments, context, kotlinDirtyFilesHolder, fsOperations, - environment, incrementalCaches + kotlinChunk, + chunk, + representativeTarget, + commonArguments, + context, + kotlinDirtyFilesHolder, + fsOperations, + environment, + incrementalCaches ) statisticsLogger.registerStatistic(chunk, System.nanoTime() - start) @@ -373,22 +416,23 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val generatedFiles = getGeneratedFiles(context, chunk, environment.outputItemsCollector) registerOutputItems(outputConsumer, generatedFiles) - representativeTarget.saveVersions(context, chunk, commonArguments) + kotlinChunk.saveVersions() - if (targets.any { hasKotlin[it] == null }) { + if (targets.any { kotlinContext.hasKotlinMarker[it] == null }) { fsOperations.markChunk(recursively = false, kotlinOnly = true, excludeFiles = kotlinDirtyFilesHolder.allDirtyFiles) } for (target in targets) { - hasKotlin[target] = true - rebuildAfterCacheVersionChanged.clean(target) + kotlinContext.hasKotlinMarker[target] = true + kotlinContext.rebuildAfterCacheVersionChanged.clean(target) } - chunk.targets.forEach { - context.kotlinBuildTargets[it]?.doAfterBuild() + kotlinChunk.targets.forEach { + it.doAfterBuild() } representativeTarget.updateChunkMappings( + context, chunk, kotlinDirtyFilesHolder, generatedFiles, @@ -407,8 +451,8 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val changesCollector = ChangesCollector() for ((target, files) in generatedFiles) { - val kotlinModuleBuilderTarget = context.kotlinBuildTargets[target]!! - kotlinModuleBuilderTarget.updateCaches(incrementalCaches[target]!!, files, changesCollector, environment) + val kotlinModuleBuilderTarget = kotlinContext.targetsBinding[target]!! + kotlinModuleBuilderTarget.updateCaches(incrementalCaches[kotlinModuleBuilderTarget]!!, files, changesCollector, environment) } updateLookupStorage(lookupTracker, dataManager, kotlinDirtyFilesHolder) @@ -426,85 +470,10 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { return OK } - private fun applyActionsOnCacheVersionChange( - actions: Set, - cacheVersionsProvider: CacheVersionProvider, - context: CompileContext, - dataManager: BuildDataManager, - targets: MutableSet, - fsOperations: FSOperationsHelper - ) { - val hasKotlin = HasKotlinMarker(dataManager) - val sortedActions = actions.sorted() - - context.testingContext?.buildLogger?.actionsOnCacheVersionChanged(sortedActions) - - for (status in sortedActions) { - when (status) { - CacheVersion.Action.REBUILD_ALL_KOTLIN -> { - markAllKotlinForRebuild(context, fsOperations, "Kotlin global lookup map format changed") - return - } - CacheVersion.Action.REBUILD_CHUNK -> { - LOG.info("Clearing caches for " + targets.joinToString { it.presentableName }) - val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) - - val kotlinBuildTargets = context.kotlinBuildTargets - for (target in targets) { - dataManager.getKotlinCache(kotlinBuildTargets[target])?.clean() - hasKotlin.clean(target) - rebuildAfterCacheVersionChanged[target] = true - } - - fsOperations.markChunk(recursively = false, kotlinOnly = true) - - return - } - CacheVersion.Action.CLEAN_NORMAL_CACHES -> { - LOG.info("Clearing caches for all targets") - - val kotlinBuildTargets = context.kotlinBuildTargets - for (target in context.allTargets()) { - dataManager.getKotlinCache(kotlinBuildTargets[target])?.clean() - } - } - CacheVersion.Action.CLEAN_DATA_CONTAINER -> { - LOG.info("Clearing lookup cache") - dataManager.cleanLookupStorage(LOG) - cacheVersionsProvider.dataContainerVersion().clean() - } - else -> { - assert(status == CacheVersion.Action.DO_NOTHING) { "Unknown version status $status" } - } - } - } - } - - private fun CompileContext.allTargets() = - projectDescriptor.buildTargetIndex.allTargets.filterIsInstanceTo>(HashSet()) - - private fun markAllKotlinForRebuild(context: CompileContext, fsOperations: FSOperationsHelper, reason: String) { - LOG.info("Rebuilding all Kotlin: $reason") - val project = context.projectDescriptor.project - val sourceRoots = project.modules.flatMap { it.sourceRoots } - val dataManager = context.projectDescriptor.dataManager - val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) - - for (sourceRoot in sourceRoots) { - val ktFiles = sourceRoot.file.walk().filter { it.isKotlinSourceFile } - fsOperations.markFiles(ktFiles.toList()) - } - - val kotlinBuildTargets = context.kotlinBuildTargets - for (target in context.allTargets()) { - dataManager.getKotlinCache(kotlinBuildTargets[target])?.clean() - rebuildAfterCacheVersionChanged[target] = true - } - - dataManager.cleanLookupStorage(LOG) - } - + // todo(1.2.80): got rid of ModuleChunk (replace with KotlinChunk) + // todo(1.2.80): introduce KotlinRoundCompileContext, move dirtyFilesHolder, fsOperations, environment to it private fun doCompileModuleChunk( + kotlinChunk: KotlinChunk, chunk: ModuleChunk, representativeTarget: KotlinModuleBuildTarget<*>, commonArguments: CommonCompilerArguments, @@ -512,9 +481,42 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { dirtyFilesHolder: KotlinDirtySourceFilesHolder, fsOperations: FSOperationsHelper, environment: JpsCompilerEnvironment, - incrementalCaches: Map + incrementalCaches: Map, JpsIncrementalCache> ): OutputItemsCollector? { + loadPlugins(representativeTarget, commonArguments, context) + kotlinChunk.targets.forEach { + it.nextRound(context) + } + + if (representativeTarget.isIncrementalCompilationEnabled) { + for (target in kotlinChunk.targets) { + val cache = incrementalCaches[target] + val jpsTarget = target.jpsModuleBuildTarget + val targetDirtyFiles = dirtyFilesHolder.byTarget[jpsTarget] + + if (cache != null && targetDirtyFiles != null) { + val complementaryFiles = cache.clearComplementaryFilesMapping( + targetDirtyFiles.dirty + targetDirtyFiles.removed + ) + + fsOperations.markFilesForCurrentRound(jpsTarget, complementaryFiles) + + cache.markDirty(targetDirtyFiles.dirty + targetDirtyFiles.removed) + } + } + } + + val isDoneSomething = representativeTarget.compileModuleChunk(chunk, commonArguments, dirtyFilesHolder, environment) + + return if (isDoneSomething) environment.outputItemsCollector else null + } + + private fun loadPlugins( + representativeTarget: KotlinModuleBuildTarget<*>, + commonArguments: CommonCompilerArguments, + context: CompileContext + ) { fun concatenate(strings: Array?, cp: List) = arrayOf(*strings.orEmpty(), *cp.toTypedArray()) for (argumentProvider in ServiceLoader.load(KotlinJpsCompilerArgumentsProvider::class.java)) { @@ -532,29 +534,12 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { LOG.debug("Plugin loaded: ${argumentProvider::class.java.simpleName}") } - - if (representativeTarget.isIncrementalCompilationEnabled) { - for (target in chunk.targets) { - val cache = incrementalCaches[target] - val targetDirtyFiles = dirtyFilesHolder.byTarget[target] - - if (cache != null && targetDirtyFiles != null) { - val complementaryFiles = cache.clearComplementaryFilesMapping(targetDirtyFiles.dirty + targetDirtyFiles.removed) - fsOperations.markFilesForCurrentRound(target, complementaryFiles) - - cache.markDirty(targetDirtyFiles.dirty + targetDirtyFiles.removed) - } - } - } - - val isDoneSomething = representativeTarget.compileModuleChunk(chunk, commonArguments, dirtyFilesHolder, environment) - - return if (isDoneSomething) environment.outputItemsCollector else null } private fun createCompileEnvironment( + context: CompileContext, kotlinModuleBuilderTarget: KotlinModuleBuildTarget<*>, - incrementalCaches: Map, + incrementalCaches: Map, JpsIncrementalCache>, lookupTracker: LookupTracker, exceptActualTracer: ExpectActualTracker, chunk: ModuleChunk, @@ -580,7 +565,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { classesToLoadByParent, messageCollector, OutputItemsCollectorImpl(), - ProgressReporterImpl(kotlinModuleBuilderTarget.context, chunk) + ProgressReporterImpl(context, chunk) ) } @@ -622,11 +607,13 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { val representativeTarget = chunk.representativeTarget() fun SimpleOutputItem.target() = - sourceFiles.firstOrNull()?.let { sourceToTarget[it] } ?: chunk.targets.singleOrNull { - it.outputDir?.let { - outputFile.startsWith(it) - } ?: false - } ?: representativeTarget + sourceFiles.firstOrNull()?.let { sourceToTarget[it] } + ?: chunk.targets.singleOrNull { target -> + target.outputDir?.let { outputDir -> + outputFile.startsWith(outputDir) + } ?: false + } + ?: representativeTarget return outputItemCollector.outputs.groupBy(SimpleOutputItem::target, SimpleOutputItem::toGeneratedFile) } @@ -697,68 +684,4 @@ private fun getLookupTracker(project: JpsProject, representativeTarget: KotlinMo if (representativeTarget.isIncrementalCompilationEnabled) return LookupTrackerImpl(testLookupTracker) return testLookupTracker -} - -private fun getIncrementalCaches( - chunk: ModuleChunk, - context: CompileContext -): Map { - val dataManager = context.projectDescriptor.dataManager - val kotlinBuildTargets = context.kotlinBuildTargets - - val chunkCaches = chunk.targets.keysToMapExceptNulls { - dataManager.getKotlinCache(kotlinBuildTargets[it]) - } - - val dependentTargets = getDependentTargets(chunkCaches.keys, context) - - val dependentCaches = dependentTargets.mapNotNull { - dataManager.getKotlinCache(kotlinBuildTargets[it]) - } - - for (chunkCache in chunkCaches.values) { - for (dependentCache in dependentCaches) { - chunkCache.addJpsDependentCache(dependentCache) - } - } - - return chunkCaches -} - -fun getDependentTargets( - compilingChunkTargets: Set, - context: CompileContext -): Set { - if (compilingChunkTargets.isEmpty()) return setOf() - - val compilingChunkModules: Set = compilingChunkTargets.mapTo(mutableSetOf()) { it.module } - val compilingChunkIsTests = compilingChunkTargets.any { it.isTests } - val classpathKind = JpsJavaClasspathKind.compile(compilingChunkIsTests) - - fun dependsOnCompilingChunk(target: BuildTarget<*>): Boolean { - if (target !is ModuleBuildTarget || compilingChunkIsTests && !target.isTests) return false - - val dependencies = getDependenciesRecursively(target.module, classpathKind) - return ContainerUtil.intersects(dependencies, compilingChunkModules) - } - - val dependentTargets = HashSet() - val sortedChunks = context.projectDescriptor.buildTargetIndex.getSortedTargetChunks(context).iterator() - - // skip chunks that are compiled before compilingChunk - while (sortedChunks.hasNext()) { - if (sortedChunks.next().targets == compilingChunkTargets) break - } - - // process chunks that compiled after compilingChunk - for (followingChunk in sortedChunks) { - if (followingChunk.targets.none(::dependsOnCompilingChunk)) continue - - dependentTargets.addAll(followingChunk.targets.filterIsInstance()) - } - - return dependentTargets -} - -private fun getDependenciesRecursively(module: JpsModule, kind: JpsJavaClasspathKind): Set = - JpsJavaExtensionService.dependencies(module).includedIn(kind).recursivelyExportedOnly().modules \ No newline at end of file +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinChunk.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinChunk.kt new file mode 100644 index 00000000000..7928975adba --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinChunk.kt @@ -0,0 +1,159 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.kotlin.config.ApiVersion +import org.jetbrains.kotlin.config.LanguageVersion +import org.jetbrains.kotlin.config.VersionView +import org.jetbrains.kotlin.incremental.storage.version.CacheStatus +import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache +import org.jetbrains.kotlin.jps.incremental.getKotlinCache +import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget +import org.jetbrains.kotlin.utils.keysToMapExceptNulls +import java.io.File + +/** + * Chunk of cyclically dependent [KotlinModuleBuildTarget]s + */ +class KotlinChunk internal constructor(val context: KotlinCompileContext, val targets: List>) { + val containsTests = targets.any { it.isTests } + + lateinit var dependencies: List + // Should be initialized only in KotlinChunk.calculateChunkDependencies + internal set + + lateinit var dependent: List + // Should be initialized only in KotlinChunk.calculateChunkDependencies + internal set + + // used only during dependency calculation + internal var _dependent: MutableSet? = mutableSetOf() + + val representativeTarget + get() = targets.first() + + private val presentableModulesToCompilersList: String + get() = targets.joinToString { "${it.module.name} (${it.globalLookupCacheId})" } + + init { + check(targets.all { it.javaClass == representativeTarget.javaClass }) { + "Cyclically dependent modules $presentableModulesToCompilersList should have same compiler." + } + } + + val compilerArguments = representativeTarget.jpsModuleBuildTarget.module.kotlinCompilerArguments + + val langVersion = + compilerArguments.languageVersion?.let { LanguageVersion.fromVersionString(it) } ?: VersionView.RELEASED_VERSION + + val apiVersion = + compilerArguments.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.createByLanguageVersion( + langVersion + ) + + fun shouldRebuild(): Boolean { + val buildMetaInfo = representativeTarget.buildMetaInfoFactory.create(compilerArguments) + + targets.forEach { target -> + if (target.isVersionChanged(this, buildMetaInfo)) { + KotlinBuilder.LOG.info("$target version changed, rebuilding $this") + return true + } + + if (target.initialLocalCacheAttributesDiff.status == CacheStatus.INVALID) { + context.testingLogger?.invalidOrUnusedCache(this, null, target.initialLocalCacheAttributesDiff) + KotlinBuilder.LOG.info("$target cache is invalid ${target.initialLocalCacheAttributesDiff}, rebuilding $this") + return true + } + } + + return false + } + + fun buildMetaInfoFile(target: ModuleBuildTarget): File = + File( + context.dataPaths.getTargetDataRoot(target), + representativeTarget.buildMetaInfoFileName + ) + + fun saveVersions() { + context.ensureLookupsCacheAttributesSaved() + + targets.forEach { + it.initialLocalCacheAttributesDiff.saveExpectedIfNeeded() + } + + val serializedMetaInfo = representativeTarget.buildMetaInfoFactory.serializeToString(compilerArguments) + + targets.forEach { + buildMetaInfoFile(it.jpsModuleBuildTarget).writeText(serializedMetaInfo) + } + } + + fun collectDependentChunksRecursivelyExportedOnly(result: MutableSet = mutableSetOf()) { + dependent.forEach { + if (result.add(it.src.chunk)) { + if (it.exported) { + it.src.chunk.collectDependentChunksRecursivelyExportedOnly(result) + } + } + } + } + + fun loadCaches(loadDependent: Boolean = true): Map, JpsIncrementalCache> { + val dataManager = context.dataManager + + val cacheByChunkTarget = targets.keysToMapExceptNulls { + dataManager.getKotlinCache(it) + } + + if (loadDependent) { + addDependentCaches(cacheByChunkTarget.values) + } + + return cacheByChunkTarget + } + + private fun addDependentCaches(targetsCaches: Collection) { + val dependentChunks = mutableSetOf() + + collectDependentChunksRecursivelyExportedOnly(dependentChunks) + + val dataManager = context.dataManager + dependentChunks.forEach { decedentChunk -> + decedentChunk.targets.forEach { + val dependentCache = dataManager.getKotlinCache(it) + if (dependentCache != null) { + + for (chunkCache in targetsCaches) { + chunkCache.addJpsDependentCache(dependentCache) + } + } + } + } + } + + /** + * The same as [org.jetbrains.jps.ModuleChunk.getPresentableShortName] + */ + val presentableShortName: String + get() = buildString { + if (containsTests) append("tests of ") + append(targets.first().module.name) + if (targets.size > 1) { + val andXMore = "and ${targets.size - 1} more" + val other = ", " + targets.asSequence().drop(1).joinToString() + append(if (other.length < andXMore.length) other else andXMore) + } + } + + override fun toString(): String { + return "KotlinChunk<${representativeTarget.javaClass.simpleName}>" + + "(${targets.joinToString { it.jpsModuleBuildTarget.presentableName }})" + } +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt new file mode 100644 index 00000000000..2abc23db574 --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinCompileContext.kt @@ -0,0 +1,240 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.FSOperations +import org.jetbrains.jps.incremental.GlobalContextKey +import org.jetbrains.jps.incremental.fs.CompilationRound +import org.jetbrains.kotlin.incremental.LookupSymbol +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff +import org.jetbrains.kotlin.incremental.storage.version.CacheStatus +import org.jetbrains.kotlin.incremental.storage.version.loadDiff +import org.jetbrains.kotlin.jps.incremental.* +import org.jetbrains.kotlin.jps.targets.KotlinTargetsIndex +import org.jetbrains.kotlin.jps.targets.KotlinTargetsIndexBuilder +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean + +/** + * KotlinCompileContext is shared between all threads (i.e. it is [GlobalContextKey]). + * + * It is initialized lazily, and only before building of first chunk with kotlin code, + * and will be disposed on build finish. + */ +internal val CompileContext.kotlin: KotlinCompileContext + get() { + val userData = getUserData(kotlinCompileContextKey) + if (userData != null) return userData + + // here is error (KotlinCompilation available only at build phase) + // let's also check for concurrent initialization + val errorMessage = "KotlinCompileContext available only at build phase " + + "(between first KotlinBuilder.chunkBuildStarted and KotlinBuilder.buildFinished)" + + synchronized(kotlinCompileContextKey) { + val newUsedData = getUserData(kotlinCompileContextKey) + if (newUsedData != null) { + error("Concurrent CompileContext.kotlin getter call and KotlinCompileContext initialization detected: $errorMessage") + } + } + + error(errorMessage) + } + +internal val kotlinCompileContextKey = GlobalContextKey("kotlin") + +class KotlinCompileContext(val jpsContext: CompileContext) { + val dataManager = jpsContext.projectDescriptor.dataManager + val dataPaths = dataManager.dataPaths + val testingLogger: TestingBuildLogger? + get() = jpsContext.testingContext?.buildLogger + + val targetsIndex: KotlinTargetsIndex = KotlinTargetsIndexBuilder(this).build() + + val targetsBinding + get() = targetsIndex.byJpsTarget + + val lookupsCacheAttributesManager: CompositeLookupsCacheAttributesManager = makeLookupsCacheAttributesManager() + + val initialLookupsCacheStateDiff: CacheAttributesDiff<*> = loadLookupsCacheStateDiff() + + val shouldCheckCacheVersions = System.getProperty(KotlinBuilder.SKIP_CACHE_VERSION_CHECK_PROPERTY) == null + + val hasKotlinMarker = HasKotlinMarker(dataManager) + + /** + * Flag to prevent rebuilding twice. + * + * TODO: looks like it is not required since cache version checking are refactored + */ + val rebuildAfterCacheVersionChanged = RebuildAfterCacheVersionChangeMarker(dataManager) + + var rebuildingAllKotlin = false + + private fun makeLookupsCacheAttributesManager(): CompositeLookupsCacheAttributesManager { + val expectedLookupsCacheComponents = mutableSetOf() + + targetsIndex.chunks.forEach { chunk -> + chunk.targets.forEach { target -> + if (target.isIncrementalCompilationEnabled) { + expectedLookupsCacheComponents.add(target.globalLookupCacheId) + } + } + } + + val lookupsCacheRootPath = dataPaths.getTargetDataRoot(KotlinDataContainerTarget) + return CompositeLookupsCacheAttributesManager(lookupsCacheRootPath, expectedLookupsCacheComponents) + } + + private fun loadLookupsCacheStateDiff(): CacheAttributesDiff { + val diff = lookupsCacheAttributesManager.loadDiff() + + if (diff.status == CacheStatus.VALID) { + // try to perform a lookup + // request rebuild if storage is corrupted + try { + dataManager.withLookupStorage { + it.get(LookupSymbol("<#NAME#>", "<#SCOPE#>")) + } + } catch (e: Exception) { + jpsReportInternalBuilderError(jpsContext, Error("Lookup storage is corrupted, probe failed: ${e.message}", e)) + markAllKotlinForRebuild("Lookup storage is corrupted") + return diff.copy(actual = null) + } + } + + return diff + } + + fun hasKotlin() = targetsIndex.chunks.any { chunk -> + chunk.targets.any { target -> + hasKotlinMarker[target] == true + } + } + + fun checkCacheVersions() { + when (initialLookupsCacheStateDiff.status) { + CacheStatus.INVALID -> { + // global cache needs to be rebuilt + + testingLogger?.invalidOrUnusedCache(null, null, initialLookupsCacheStateDiff) + + if (initialLookupsCacheStateDiff.actual != null) { + markAllKotlinForRebuild("Kotlin incremental cache settings or format was changed") + clearLookupCache() + } else { + markAllKotlinForRebuild("Kotlin incremental cache is missed or corrupted") + } + } + CacheStatus.VALID -> Unit + CacheStatus.SHOULD_BE_CLEARED -> { + jpsContext.testingContext?.buildLogger?.invalidOrUnusedCache(null, null, initialLookupsCacheStateDiff) + KotlinBuilder.LOG.info("Removing global cache as it is not required anymore: $initialLookupsCacheStateDiff") + + clearAllCaches() + } + CacheStatus.CLEARED -> Unit + } + } + + private val lookupAttributesSaved = AtomicBoolean(false) + + /** + * Called on every successful compilation + */ + fun ensureLookupsCacheAttributesSaved() { + if (lookupAttributesSaved.compareAndSet(false, true)) { + initialLookupsCacheStateDiff.saveExpectedIfNeeded() + } + } + + fun checkChunkCacheVersion(chunk: KotlinChunk) { + if (shouldCheckCacheVersions && !rebuildingAllKotlin) { + if (chunk.shouldRebuild()) markChunkForRebuildBeforeBuild(chunk) + } + } + + private fun logMarkDirtyForTestingBeforeRound(file: File, shouldProcess: Boolean): Boolean { + if (shouldProcess) { + testingLogger?.markedAsDirtyBeforeRound(listOf(file)) + } + return shouldProcess + } + + private fun markAllKotlinForRebuild(reason: String) { + if (rebuildingAllKotlin) return + rebuildingAllKotlin = true + + KotlinBuilder.LOG.info("Rebuilding all Kotlin: $reason") + + val dataManager = jpsContext.projectDescriptor.dataManager + + targetsIndex.chunks.forEach { + markChunkForRebuildBeforeBuild(it) + } + + dataManager.cleanLookupStorage(KotlinBuilder.LOG) + } + + private fun markChunkForRebuildBeforeBuild(chunk: KotlinChunk) { + chunk.targets.forEach { + FSOperations.markDirty(jpsContext, CompilationRound.NEXT, it.jpsModuleBuildTarget) { file -> + logMarkDirtyForTestingBeforeRound(file, file.isKotlinSourceFile) + } + + dataManager.getKotlinCache(it)?.clean() + hasKotlinMarker.clean(it) + rebuildAfterCacheVersionChanged[it] = true + } + } + + private fun clearAllCaches() { + clearLookupCache() + + KotlinBuilder.LOG.info("Clearing caches for all targets") + targetsIndex.chunks.forEach { chunk -> + chunk.targets.forEach { + dataManager.getKotlinCache(it)?.clean() + } + } + } + + private fun clearLookupCache() { + KotlinBuilder.LOG.info("Clearing lookup cache") + dataManager.cleanLookupStorage(KotlinBuilder.LOG) + initialLookupsCacheStateDiff.saveExpectedIfNeeded() + } + + fun cleanupCaches() { + // todo: remove lookups for targets with disabled IC (or split global lookups cache into several caches for each compiler) + + targetsIndex.chunks.forEach { chunk -> + chunk.targets.forEach { target -> + if (target.initialLocalCacheAttributesDiff.status == CacheStatus.SHOULD_BE_CLEARED) { + KotlinBuilder.LOG.info( + "$target caches is cleared as not required anymore: ${target.initialLocalCacheAttributesDiff}" + ) + testingLogger?.invalidOrUnusedCache(null, target, target.initialLocalCacheAttributesDiff) + dataManager.getKotlinCache(target)?.clean() + } + } + } + } + + fun dispose() { + + } + + fun getChunk(rawChunk: ModuleChunk): KotlinChunk? { + val rawRepresentativeTarget = rawChunk.representativeTarget() + if (rawRepresentativeTarget !in targetsBinding) return null + + return targetsIndex.chunksByJpsRepresentativeTarget[rawRepresentativeTarget] + ?: error("Kotlin binding for chunk $this is not loaded at build start") + } +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinDirtySourceFilesHolder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinDirtySourceFilesHolder.kt index 7665579f765..176ea680d9b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinDirtySourceFilesHolder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinDirtySourceFilesHolder.kt @@ -48,7 +48,7 @@ class KotlinDirtySourceFilesHolder( * and during KotlinDirtySourceFilesHolder initialization. */ internal fun _markDirty(file: File) { - _dirty.add(file) + _dirty.add(file.canonicalFile) } } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinSourceRootProvider.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinSourceRootProvider.kt index a11453753af..a0dc46d6bb7 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinSourceRootProvider.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinSourceRootProvider.kt @@ -55,17 +55,12 @@ class KotlinSourceRootProvider : AdditionalRootsProviderService - addModuleSourceRoots(result, sourceSetModule, target, false) + addModuleSourceRoots(result, sourceSetModule, target) } // legacy multiplatform model support: module.expectedByModules.forEach { commonModule -> - // At the moment, incremental compilation is not supported by K2Metadata compiler. - // To avoid long running compilation of common modules, we do not run K2Metadata at all: - // instead all the common source roots are transitively added to the final platform modules. - val isRecursive = true - - addModuleSourceRoots(result, commonModule, target, isRecursive) + addModuleSourceRoots(result, commonModule, target) } return result @@ -74,8 +69,7 @@ class KotlinSourceRootProvider : AdditionalRootsProviderService, module: JpsModule, - target: ModuleBuildTarget, - recursive: Boolean = false + target: ModuleBuildTarget ) { for (commonSourceRoot in module.sourceRoots) { val isCommonTestsRootType = commonSourceRoot.rootType.isTestsRootType @@ -94,21 +88,6 @@ class KotlinSourceRootProvider : AdditionalRootsProviderService): Boolean? = + get(target.jpsModuleBuildTarget) + operator fun get(target: ModuleBuildTarget): Boolean? { val file = target.markerFile @@ -34,6 +38,9 @@ abstract class MarkerFile(private val fileName: String, private val paths: Build return file.readText().toBoolean() } + operator fun set(target: KotlinModuleBuildTarget<*>, value: Boolean) = + set(target.jpsModuleBuildTarget, value) + operator fun set(target: ModuleBuildTarget, value: Boolean) { val file = target.markerFile @@ -45,6 +52,9 @@ abstract class MarkerFile(private val fileName: String, private val paths: Build file.writeText(value.toString()) } + fun clean(target: KotlinModuleBuildTarget<*>) = + clean(target.jpsModuleBuildTarget) + fun clean(target: ModuleBuildTarget) { target.markerFile.delete() } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/MessageCollectorAdapter.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/MessageCollectorAdapter.kt index 2bdb4e8c217..5c0917a0739 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/MessageCollectorAdapter.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/MessageCollectorAdapter.kt @@ -12,7 +12,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.config.CompilerRunnerConstants -import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import java.io.File class MessageCollectorAdapter( diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/BuildLogger.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingBuildLogger.kt similarity index 64% rename from jps-plugin/src/org/jetbrains/kotlin/jps/build/BuildLogger.kt rename to jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingBuildLogger.kt index 654fe2ab4ee..d500f50b0e5 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/BuildLogger.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingBuildLogger.kt @@ -16,16 +16,19 @@ package org.jetbrains.kotlin.jps.build -import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.jps.incremental.ModuleLevelBuilder -import org.jetbrains.kotlin.incremental.CacheVersion +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import java.io.File -interface BuildLogger { - fun actionsOnCacheVersionChanged(actions: List) - fun buildStarted(context: CompileContext, chunk: ModuleChunk) - fun afterBuildStarted(context: CompileContext, chunk: ModuleChunk) +/** + * Used for assertions in tests. + */ +interface TestingBuildLogger { + fun invalidOrUnusedCache(chunk: KotlinChunk?, target: KotlinModuleBuildTarget<*>?, attributesDiff: CacheAttributesDiff<*>) + fun chunkBuildStarted(context: CompileContext, chunk: org.jetbrains.jps.ModuleChunk) + fun afterChunkBuildStarted(context: CompileContext, chunk: org.jetbrains.jps.ModuleChunk) fun buildFinished(exitCode: ModuleLevelBuilder.ExitCode) fun markedAsDirtyBeforeRound(files: Iterable) fun markedAsDirtyAfterRound(files: Iterable) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingContext.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingContext.kt index 3f735bdf2c6..310f89c667b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingContext.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/TestingContext.kt @@ -24,7 +24,7 @@ import org.jetbrains.jps.model.JpsSimpleElement import org.jetbrains.jps.model.ex.JpsElementChildRoleBase import org.jetbrains.kotlin.incremental.components.LookupTracker -private val TESTING_CONTEXT = JpsElementChildRoleBase.create>("Testing context") +private val TESTING_CONTEXT = JpsElementChildRoleBase.create>("Testing kcontext") @TestOnly fun JpsProject.setTestingContext(context: TestingContext) { @@ -40,5 +40,7 @@ val CompileContext.testingContext: TestingContext? class TestingContext( val lookupTracker: LookupTracker, - val buildLogger: BuildLogger -) + val buildLogger: TestingBuildLogger +) { + var kotlinCompileContext: KotlinCompileContext? = null +} diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt new file mode 100644 index 00000000000..cd15ac5eeed --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + val builderError = CompilerMessage.createInternalBuilderError("Kotlin", error) + context.processMessage(builderError) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.173 b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.173 new file mode 100644 index 00000000000..c014f4b30fa --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.173 @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + KotlinBuilder.LOG.info(error) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.181 b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.181 new file mode 100644 index 00000000000..c014f4b30fa --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.181 @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + KotlinBuilder.LOG.info(error) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as31 b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as31 new file mode 100644 index 00000000000..c014f4b30fa --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as31 @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + KotlinBuilder.LOG.info(error) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as32 b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as32 new file mode 100644 index 00000000000..c014f4b30fa --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as32 @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + KotlinBuilder.LOG.info(error) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as33 b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as33 new file mode 100644 index 00000000000..c014f4b30fa --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/ideaPlatform.kt.as33 @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.build + +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.messages.CompilerMessage + +fun jpsReportInternalBuilderError(context: CompileContext, error: Throwable) { + KotlinBuilder.LOG.info(error) +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/jpsUtil.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/jpsUtil.kt index 3f50e40d2df..c63f0771e6a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/jpsUtil.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/jpsUtil.kt @@ -17,9 +17,31 @@ package org.jetbrains.kotlin.jps.build import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.jps.model.module.JpsModule fun ModuleChunk.isDummy(context: CompileContext): Boolean { val targetIndex = context.projectDescriptor.buildTargetIndex return targets.all { targetIndex.isDummy(it) } -} \ No newline at end of file +} + +@Deprecated("Use `kotlin.targetBinding` instead", ReplaceWith("kotlin.targetsBinding")) +val CompileContext.kotlinBuildTargets + get() = kotlin.targetsBinding + +fun ModuleChunk.toKotlinChunk(context: CompileContext): KotlinChunk? = + context.kotlin.getChunk(this) + +fun ModuleBuildTarget(module: JpsModule, isTests: Boolean) = + ModuleBuildTarget( + module, + if (isTests) JavaModuleBuildTargetType.TEST else JavaModuleBuildTargetType.PRODUCTION + ) + +val JpsModule.productionBuildTarget + get() = ModuleBuildTarget(this, false) + +val JpsModule.testBuildTarget + get() = ModuleBuildTarget(this, true) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersionProvider.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersionProvider.kt deleted file mode 100644 index da25b6c95b4..00000000000 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersionProvider.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.jps.incremental - -import org.jetbrains.jps.builders.BuildTarget -import org.jetbrains.jps.builders.storage.BuildDataPaths -import org.jetbrains.jps.incremental.ModuleBuildTarget -import org.jetbrains.kotlin.incremental.CacheVersion -import org.jetbrains.kotlin.incremental.dataContainerCacheVersion -import org.jetbrains.kotlin.incremental.normalCacheVersion -import java.io.File - - -class CacheVersionProvider( - private val paths: BuildDataPaths, - private val isIncrementalCompilationEnabled: Boolean -) { - private val BuildTarget<*>.dataRoot: File - get() = paths.getTargetDataRoot(this) - - fun normalVersion(target: ModuleBuildTarget): CacheVersion = normalCacheVersion(target.dataRoot, isIncrementalCompilationEnabled) - - fun dataContainerVersion(): CacheVersion = dataContainerCacheVersion(KotlinDataContainerTarget.dataRoot, isIncrementalCompilationEnabled) - - fun allVersions(targets: Iterable): Iterable { - val versions = arrayListOf() - versions.add(dataContainerCacheVersion(KotlinDataContainerTarget.dataRoot, isIncrementalCompilationEnabled)) - - for (dataRoot in targets.map { it.dataRoot }) { - versions.add(normalCacheVersion(dataRoot, isIncrementalCompilationEnabled)) - } - - return versions - } -} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributes.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributes.kt new file mode 100644 index 00000000000..e50c18e9471 --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CompositeLookupsCacheAttributes.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.incremental + +import org.jetbrains.annotations.TestOnly +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesManager +import org.jetbrains.kotlin.incremental.storage.version.CacheVersion +import org.jetbrains.kotlin.incremental.storage.version.lookupsCacheVersionManager +import java.io.File +import java.io.IOException + +/** + * Attributes manager for global lookups cache that may contain lookups for several compilers (jvm, js). + * Works by delegating to [lookupsCacheVersionManager] and managing additional file with list of executed compilers (cache components). + * + * TODO(1.2.80): got rid of shared lookup cache, replace with individual lookup cache for each compiler + */ +class CompositeLookupsCacheAttributesManager( + rootPath: File, + expectedComponents: Set +) : CacheAttributesManager { + private val versionManager = lookupsCacheVersionManager( + rootPath, + expectedComponents.isNotEmpty() + ) + + private val actualComponentsFile = File(rootPath, "components.txt") + + override val expected: CompositeLookupsCacheAttributes? = + if (expectedComponents.isEmpty()) null + else CompositeLookupsCacheAttributes(versionManager.expected!!.version, expectedComponents) + + override fun loadActual(): CompositeLookupsCacheAttributes? { + val version = versionManager.loadActual() ?: return null + + if (!actualComponentsFile.exists()) return null + + val components = try { + actualComponentsFile.readLines().toSet() + } catch (e: IOException) { + return null + } + + return CompositeLookupsCacheAttributes(version.version, components) + } + + override fun writeActualVersion(values: CompositeLookupsCacheAttributes?) { + if (values == null) { + versionManager.writeActualVersion(null) + actualComponentsFile.delete() + } else { + versionManager.writeActualVersion(CacheVersion(values.version)) + + actualComponentsFile.parentFile.mkdirs() + actualComponentsFile.writeText(values.components.joinToString("\n")) + } + } + + override fun isCompatible(actual: CompositeLookupsCacheAttributes, expected: CompositeLookupsCacheAttributes): Boolean { + // cache can be reused when all required (expected) components are present + // (components that are not required anymore are not not interfere) + return actual.version == expected.version && actual.components.containsAll(expected.components) + } + + @get:TestOnly + val versionManagerForTesting + get() = versionManager +} + +data class CompositeLookupsCacheAttributes( + val version: Int, + val components: Set +) { + override fun toString() = "($version, $components)" +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt index 52875842765..3d999048d4e 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt @@ -25,7 +25,7 @@ import org.jetbrains.kotlin.incremental.IncrementalCacheCommon import org.jetbrains.kotlin.incremental.IncrementalJsCache import org.jetbrains.kotlin.incremental.IncrementalJvmCache import org.jetbrains.kotlin.jps.build.KotlinBuilder -import org.jetbrains.kotlin.jps.platforms.KotlinModuleBuildTarget +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import java.io.File interface JpsIncrementalCache : IncrementalCacheCommon, StorageOwner { diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/TargetPlatform.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/TargetPlatform.kt deleted file mode 100644 index 94f9ddfdf97..00000000000 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/TargetPlatform.kt +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.jps.platforms - -import com.intellij.openapi.util.Key -import org.jetbrains.jps.builders.ModuleBasedBuildTargetType -import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType -import org.jetbrains.jps.incremental.CompileContext -import org.jetbrains.jps.incremental.ModuleBuildTarget -import org.jetbrains.jps.model.library.JpsOrderRootType -import org.jetbrains.jps.model.module.JpsModule -import org.jetbrains.jps.util.JpsPathUtil -import org.jetbrains.kotlin.jps.model.platform -import org.jetbrains.kotlin.platform.DefaultIdeTargetPlatformKindProvider -import org.jetbrains.kotlin.platform.IdePlatformKind -import org.jetbrains.kotlin.platform.impl.* -import org.jetbrains.kotlin.utils.LibraryUtils -import java.util.concurrent.ConcurrentHashMap - -fun ModuleBuildTarget(module: JpsModule, isTests: Boolean) = - ModuleBuildTarget(module, if (isTests) JavaModuleBuildTargetType.TEST else JavaModuleBuildTargetType.PRODUCTION) - -val JpsModule.productionBuildTarget - get() = ModuleBuildTarget(this, false) - -private val kotlinBuildTargetsCompileContextKey = Key("kotlinBuildTargets") - -val CompileContext.kotlinBuildTargets: KotlinBuildTargets - get() { - val value = getUserData(kotlinBuildTargetsCompileContextKey) - if (value != null) return value - - synchronized(this) { - val actualValue = getUserData(kotlinBuildTargetsCompileContextKey) - if (actualValue != null) return actualValue - - val newValue = KotlinBuildTargets(this) - putUserData(kotlinBuildTargetsCompileContextKey, newValue) - return newValue - } - } - -class KotlinBuildTargets internal constructor(val compileContext: CompileContext) { - private val byJpsModuleBuildTarget = ConcurrentHashMap>() - private val isKotlinJsStdlibJar = ConcurrentHashMap() - - @JvmName("getNullable") - operator fun get(target: ModuleBuildTarget?): KotlinModuleBuildTarget<*>? { - if (target == null) return null - return get(target) - } - - operator fun get(target: ModuleBuildTarget): KotlinModuleBuildTarget<*>? { - if (target.targetType !is ModuleBasedBuildTargetType) return null - - return byJpsModuleBuildTarget.computeIfAbsent(target) { - val platform = target.module.platform?.kind ?: detectTargetPlatform(target) - - when { - platform.isCommon -> KotlinCommonModuleBuildTarget(compileContext, target) - platform.isJavaScript -> KotlinJsModuleBuildTarget(compileContext, target) - platform.isJvm -> KotlinJvmModuleBuildTarget(compileContext, target) - else -> error("Invalid platform $platform") - } - } - } - - /** - * Compatibility for KT-14082 - * todo: remove when all projects migrated to facets - */ - private fun detectTargetPlatform(target: ModuleBuildTarget): IdePlatformKind<*> { - if (hasJsStdLib(target)) { - return JsIdePlatformKind - } - - return JvmIdePlatformKind - return DefaultIdeTargetPlatformKindProvider.defaultPlatform.kind - } - - private fun hasJsStdLib(target: ModuleBuildTarget): Boolean { - KotlinJvmModuleBuildTarget(compileContext, target).allDependencies.libraries.forEach { library -> - for (root in library.getRoots(JpsOrderRootType.COMPILED)) { - val url = root.url - - val isKotlinJsLib = isKotlinJsStdlibJar.computeIfAbsent(url) { - LibraryUtils.isKotlinJavascriptStdLibrary(JpsPathUtil.urlToFile(url)) - } - - if (isKotlinJsLib) return true - } - } - - return false - } -} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinCommonModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinCommonModuleBuildTarget.kt similarity index 87% rename from jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinCommonModuleBuildTarget.kt rename to jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinCommonModuleBuildTarget.kt index fada07f934c..997912116c5 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinCommonModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinCommonModuleBuildTarget.kt @@ -3,11 +3,10 @@ * that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.jps.platforms +package org.jetbrains.kotlin.jps.targets import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.builders.storage.BuildDataPaths -import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.model.library.JpsOrderRootType import org.jetbrains.jps.model.module.JpsModule @@ -17,15 +16,16 @@ import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner -import org.jetbrains.kotlin.config.IncrementalCompilation +import org.jetbrains.kotlin.jps.build.KotlinCompileContext import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder +import org.jetbrains.kotlin.jps.build.ModuleBuildTarget import org.jetbrains.kotlin.jps.model.k2MetadataCompilerArguments import org.jetbrains.kotlin.jps.model.kotlinCompilerSettings private const val COMMON_BUILD_META_INFO_FILE_NAME = "common-build-meta-info.txt" -class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : - KotlinModuleBuildTarget(context, jpsModuleBuildTarget) { +class KotlinCommonModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : + KotlinModuleBuildTarget(kotlinContext, jpsModuleBuildTarget) { override val isIncrementalCompilationEnabled: Boolean get() = false @@ -36,6 +36,9 @@ class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarge override val buildMetaInfoFileName get() = COMMON_BUILD_META_INFO_FILE_NAME + override val globalLookupCacheId: String + get() = "metadata-compiler" + override fun compileModuleChunk( chunk: ModuleChunk, commonArguments: CommonCompilerArguments, @@ -87,7 +90,7 @@ class KotlinCommonModuleBuildTarget(context: CompileContext, jpsModuleBuildTarge result: MutableList, isTests: Boolean ) { - val dependencyBuildTarget = context.kotlinBuildTargets[ModuleBuildTarget(module, isTests)] + val dependencyBuildTarget = kotlinContext.targetsBinding[ModuleBuildTarget(module, isTests)] if (dependencyBuildTarget != this@KotlinCommonModuleBuildTarget && dependencyBuildTarget is KotlinCommonModuleBuildTarget && diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJsModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt similarity index 91% rename from jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJsModuleBuildTarget.kt rename to jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt index c650f587b41..593deeb618a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJsModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJsModuleBuildTarget.kt @@ -3,11 +3,10 @@ * that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.jps.platforms +package org.jetbrains.kotlin.jps.targets import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.builders.storage.BuildDataPaths -import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.model.library.JpsOrderRootType import org.jetbrains.jps.model.module.JpsModule @@ -27,7 +26,9 @@ import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.incremental.js.IncrementalDataProviderFromCache import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl +import org.jetbrains.kotlin.jps.build.KotlinCompileContext import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder +import org.jetbrains.kotlin.jps.build.ModuleBuildTarget import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJsCache import org.jetbrains.kotlin.jps.model.k2JsCompilerArguments @@ -42,8 +43,10 @@ import java.net.URI private const val JS_BUILD_META_INFO_FILE_NAME = "js-build-meta-info.txt" -class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : - KotlinModuleBuildTarget(compileContext, jpsModuleBuildTarget) { +class KotlinJsModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : + KotlinModuleBuildTarget(kotlinContext, jpsModuleBuildTarget) { + override val globalLookupCacheId: String + get() = "js" override val isIncrementalCompilationEnabled: Boolean get() = IncrementalCompilation.isEnabledForJs() @@ -56,13 +59,13 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa val isFirstBuild: Boolean get() { - val targetDataRoot = context.projectDescriptor.dataManager.dataPaths.getTargetDataRoot(jpsModuleBuildTarget) + val targetDataRoot = jpsGlobalContext.projectDescriptor.dataManager.dataPaths.getTargetDataRoot(jpsModuleBuildTarget) return !IncrementalJsCache.hasHeaderFile(targetDataRoot) } override fun makeServices( builder: Services.Builder, - incrementalCaches: Map, + incrementalCaches: Map, JpsIncrementalCache>, lookupTracker: LookupTracker, exceptActualTracer: ExpectActualTracker ) { @@ -72,7 +75,7 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl()) if (isIncrementalCompilationEnabled && !isFirstBuild) { - val cache = incrementalCaches[jpsModuleBuildTarget] as IncrementalJsCache + val cache = incrementalCaches[this@KotlinJsModuleBuildTarget] as IncrementalJsCache register( IncrementalDataProvider::class.java, @@ -188,7 +191,7 @@ class KotlinJsModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTa result: MutableList, isTests: Boolean ) { - val dependencyBuildTarget = context.kotlinBuildTargets[ModuleBuildTarget(module, isTests)] + val dependencyBuildTarget = kotlinContext.targetsBinding[ModuleBuildTarget(module, isTests)] if (dependencyBuildTarget != this@KotlinJsModuleBuildTarget && dependencyBuildTarget is KotlinJsModuleBuildTarget && diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJvmModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt similarity index 90% rename from jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJvmModuleBuildTarget.kt rename to jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt index 2d9a2958163..bedf425fce1 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinJvmModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinJvmModuleBuildTarget.kt @@ -3,7 +3,7 @@ * that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.jps.platforms +package org.jetbrains.kotlin.jps.targets import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.text.StringUtil @@ -32,6 +32,7 @@ import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.jps.build.KotlinBuilder +import org.jetbrains.kotlin.jps.build.KotlinCompileContext import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache import org.jetbrains.kotlin.jps.incremental.JpsIncrementalJvmCache @@ -48,8 +49,8 @@ import java.io.IOException private const val JVM_BUILD_META_INFO_FILE_NAME = "jvm-build-meta-info.txt" -class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : - KotlinModuleBuildTarget(compileContext, jpsModuleBuildTarget) { +class KotlinJvmModuleBuildTarget(kotlinContext: KotlinCompileContext, jpsModuleBuildTarget: ModuleBuildTarget) : + KotlinModuleBuildTarget(kotlinContext, jpsModuleBuildTarget) { override val isIncrementalCompilationEnabled: Boolean get() = IncrementalCompilation.isEnabledForJvm() @@ -64,7 +65,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT override fun makeServices( builder: Services.Builder, - incrementalCaches: Map, + incrementalCaches: Map, JpsIncrementalCache>, lookupTracker: LookupTracker, exceptActualTracer: ExpectActualTracker ) { @@ -74,7 +75,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT register( IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl( - incrementalCaches.mapKeys { context.kotlinBuildTargets[it.key]!!.targetId } as Map + incrementalCaches.mapKeys { it.key.targetId } as Map ) ) } @@ -141,7 +142,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT var hasDirtySources = false - val targets = chunk.targets.mapNotNull { this.context.kotlinBuildTargets[it] as? KotlinJvmModuleBuildTarget } + val targets = chunk.targets.mapNotNull { kotlinContext.targetsBinding[it] as? KotlinJvmModuleBuildTarget } val outputDirs = targets.map { it.outputDir }.toSet() @@ -159,7 +160,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT kotlinModuleId.name, outputDir.absolutePath, moduleSources, - target.findSourceRoots(context), + target.findSourceRoots(dirtyFilesHolder.context), target.findClassPathRoots(), commonSources, target.findModularJdkRoot(), @@ -256,14 +257,18 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT updateIncrementalCache(files, jpsIncrementalCache as IncrementalJvmCache, changesCollector, null) } + override val globalLookupCacheId: String + get() = "jvm" + override fun updateChunkMappings( + localContext: CompileContext, chunk: ModuleChunk, dirtyFilesHolder: KotlinDirtySourceFilesHolder, outputItems: Map>, - incrementalCaches: Map + incrementalCaches: Map, JpsIncrementalCache> ) { - val previousMappings = context.projectDescriptor.dataManager.mappings - val callback = JavaBuilderUtil.getDependenciesRegistrar(context) + val previousMappings = localContext.projectDescriptor.dataManager.mappings + val callback = JavaBuilderUtil.getDependenciesRegistrar(localContext) val targetDirtyFiles: Map> = chunk.targets.keysToMap { val files = HashSet() @@ -273,7 +278,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT } fun getOldSourceFiles(target: ModuleBuildTarget, generatedClass: GeneratedJvmClass): Set { - val cache = incrementalCaches[target] ?: return emptySet() + val cache = incrementalCaches[kotlinContext.targetsBinding[target]] ?: return emptySet() cache as JpsIncrementalJvmCache val className = generatedClass.outputClass.className @@ -301,7 +306,7 @@ class KotlinJvmModuleBuildTarget(compileContext: CompileContext, jpsModuleBuildT } val allCompiled = dirtyFilesHolder.allDirtyFiles - JavaBuilderUtil.registerFilesToCompile(context, allCompiled) - JavaBuilderUtil.registerSuccessfullyCompiled(context, allCompiled) + JavaBuilderUtil.registerFilesToCompile(localContext, allCompiled) + JavaBuilderUtil.registerSuccessfullyCompiled(localContext, allCompiled) } } \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinModuleBuildTarget.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt similarity index 64% rename from jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinModuleBuildTarget.kt rename to jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt index 678bd491ec3..1002fa556ef 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/platforms/KotlinModuleBuildTarget.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinModuleBuildTarget.kt @@ -3,14 +3,13 @@ * that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.jps.platforms +package org.jetbrains.kotlin.jps.targets import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.builders.storage.BuildDataPaths import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.incremental.ProjectBuildException -import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.model.java.JpsJavaClasspathKind import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.model.module.JpsModule @@ -21,20 +20,18 @@ import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.compilerRunner.JpsCompilerEnvironment -import org.jetbrains.kotlin.config.* -import org.jetbrains.kotlin.incremental.CacheVersion +import org.jetbrains.kotlin.config.ApiVersion +import org.jetbrains.kotlin.config.LanguageVersion +import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.ChangesCollector import org.jetbrains.kotlin.incremental.ExpectActualTrackerImpl -import org.jetbrains.kotlin.incremental.LookupTrackerImpl import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.jps.build.KotlinBuilder -import org.jetbrains.kotlin.jps.build.KotlinIncludedModuleSourceRoot -import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder -import org.jetbrains.kotlin.jps.build.isKotlinSourceFile -import org.jetbrains.kotlin.jps.incremental.CacheVersionProvider +import org.jetbrains.kotlin.incremental.storage.version.CacheAttributesDiff +import org.jetbrains.kotlin.incremental.storage.version.loadDiff +import org.jetbrains.kotlin.incremental.storage.version.localCacheVersionManager +import org.jetbrains.kotlin.jps.build.* import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache -import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments import org.jetbrains.kotlin.jps.model.productionOutputFilePath import org.jetbrains.kotlin.jps.model.testOutputFilePath import org.jetbrains.kotlin.modules.TargetId @@ -46,12 +43,35 @@ import java.io.File /** * Properties and actions for Kotlin test / production module build target. */ -abstract class KotlinModuleBuildTarget( - val context: CompileContext, +abstract class KotlinModuleBuildTarget internal constructor( + val kotlinContext: KotlinCompileContext, val jpsModuleBuildTarget: ModuleBuildTarget ) { + /** + * Note: beware of using this context for getting compilation round dependent data: + * for example groovy can provide temp source roots with stubs, and it will be visible + * only in round local compile context. + * + * TODO(1.2.80): got rid of jpsGlobalContext and replace it with kotlinContext + */ + val jpsGlobalContext: CompileContext + get() = kotlinContext.jpsContext + + // Initialized in KotlinCompileContext.loadTargets + lateinit var chunk: KotlinChunk + + abstract val globalLookupCacheId: String + abstract val isIncrementalCompilationEnabled: Boolean + @Suppress("LeakingThis") + val localCacheVersionManager = localCacheVersionManager( + kotlinContext.dataPaths.getTargetDataRoot(jpsModuleBuildTarget), + isIncrementalCompilationEnabled + ) + + val initialLocalCacheAttributesDiff: CacheAttributesDiff<*> = localCacheVersionManager.loadDiff() + val module: JpsModule get() = jpsModuleBuildTarget.module @@ -76,8 +96,8 @@ abstract class KotlinModuleBuildTarget( val explicitOutputPath = if (isTests) module.testOutputFilePath else module.productionOutputFilePath val explicitOutputDir = explicitOutputPath?.let { File(it).absoluteFile.parentFile } return@lazy explicitOutputDir - ?: jpsModuleBuildTarget.outputDir - ?: throw ProjectBuildException("No output directory found for " + this) + ?: jpsModuleBuildTarget.outputDir + ?: throw ProjectBuildException("No output directory found for " + this) } val friendBuildTargets: List> @@ -85,8 +105,8 @@ abstract class KotlinModuleBuildTarget( val result = mutableListOf>() if (isTests) { - result.addIfNotNull(context.kotlinBuildTargets[module.productionBuildTarget]) - result.addIfNotNull(context.kotlinBuildTargets[relatedProductionModule?.productionBuildTarget]) + result.addIfNotNull(kotlinContext.targetsBinding[module.productionBuildTarget]) + result.addIfNotNull(kotlinContext.targetsBinding[relatedProductionModule?.productionBuildTarget]) } return result.filter { it.sources.isNotEmpty() } @@ -100,26 +120,48 @@ abstract class KotlinModuleBuildTarget( private val relatedProductionModule: JpsModule? get() = JpsJavaExtensionService.getInstance().getTestModuleProperties(module)?.productionModule + data class Dependency( + val src: KotlinModuleBuildTarget<*>, + val target: KotlinModuleBuildTarget<*>, + val exported: Boolean + ) + + // TODO(1.2.80): try replace allDependencies with KotlinChunk.collectDependentChunksRecursivelyExportedOnly + @Deprecated("Consider using precalculated KotlinChunk.collectDependentChunksRecursivelyExportedOnly") val allDependencies by lazy { JpsJavaExtensionService.dependencies(module).recursively().exportedOnly() .includedIn(JpsJavaClasspathKind.compile(isTests)) } - val sources: Map by lazy { - mutableMapOf().also { result -> - collectSources(result) + /** + * All sources of this target (including non dirty). + * Initialized lazily based on global context and will be updated on each round based on round local context. + * + * Update required since source roots can be changed, for example groovy can provide new temporary source roots with stubs. + * Lazy initialization is required for friend build targets, when friends are not compiled in this build run. + */ + val sources: Map + get() = _sources ?: synchronized(this) { + _sources ?: updateSourcesList(jpsGlobalContext) } + + @Volatile + private var _sources: Map? = null + + fun nextRound(localContext: CompileContext) { + updateSourcesList(localContext) } - private fun collectSources(receiver: MutableMap) { + private fun updateSourcesList(localContext: CompileContext): Map { + val result = mutableMapOf() val moduleExcludes = module.excludeRootsList.urls.mapTo(java.util.HashSet(), JpsPathUtil::urlToFile) val compilerExcludes = JpsJavaExtensionService.getInstance() .getOrCreateCompilerConfiguration(module.project) .compilerExcludes - val buildRootIndex = context.projectDescriptor.buildRootIndex - val roots = buildRootIndex.getTargetRoots(jpsModuleBuildTarget, context) + val buildRootIndex = localContext.projectDescriptor.buildRootIndex + val roots = buildRootIndex.getTargetRoots(jpsModuleBuildTarget, localContext) roots.forEach { rootDescriptor -> val isIncludedSourceRoot = rootDescriptor is KotlinIncludedModuleSourceRoot @@ -127,11 +169,14 @@ abstract class KotlinModuleBuildTarget( .onEnter { file -> file !in moduleExcludes } .forEach { file -> if (!compilerExcludes.isExcluded(file) && file.isFile && file.isKotlinSourceFile) { - receiver[file] = Source(file, isIncludedSourceRoot) + result[file] = Source(file, isIncludedSourceRoot) } } } + + this._sources = result + return result } /** @@ -179,9 +224,6 @@ abstract class KotlinModuleBuildTarget( return false } - fun compilerArgumentsForChunk(chunk: ModuleChunk): CommonCompilerArguments = - chunk.representativeTarget().module.kotlinCompilerArguments - open fun doAfterBuild() { } @@ -193,10 +235,11 @@ abstract class KotlinModuleBuildTarget( * Called for `ModuleChunk.representativeTarget` */ open fun updateChunkMappings( + localContext: CompileContext, chunk: ModuleChunk, dirtyFilesHolder: KotlinDirtySourceFilesHolder, outputItems: Map>, - incrementalCaches: Map + incrementalCaches: Map, JpsIncrementalCache> ) { // by default do nothing } @@ -213,7 +256,7 @@ abstract class KotlinModuleBuildTarget( open fun makeServices( builder: Services.Builder, - incrementalCaches: Map, + incrementalCaches: Map, JpsIncrementalCache>, lookupTracker: LookupTracker, exceptActualTracer: ExpectActualTracker ) { @@ -222,7 +265,7 @@ abstract class KotlinModuleBuildTarget( register(ExpectActualTracker::class.java, exceptActualTracer) register(CompilationCanceledStatus::class.java, object : CompilationCanceledStatus { override fun checkCanceled() { - if (context.cancelStatus.isCanceled) throw CompilationCanceledException() + if (jpsGlobalContext.cancelStatus.isCanceled) throw CompilationCanceledException() } }) } @@ -261,7 +304,7 @@ abstract class KotlinModuleBuildTarget( val hasRemovedSources = dirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget).isNotEmpty() val hasDirtyOrRemovedSources = moduleSources.isNotEmpty() || hasRemovedSources if (hasDirtyOrRemovedSources) { - val logger = context.loggingManager.projectBuilderLogger + val logger = jpsGlobalContext.loggingManager.projectBuilderLogger if (logger.isEnabled) { logger.logCompiledFiles(moduleSources, KotlinBuilder.KOTLIN_BUILDER_NAME, "Compiling files:") } @@ -274,66 +317,48 @@ abstract class KotlinModuleBuildTarget( abstract val buildMetaInfoFileName: String - fun buildMetaInfoFile(target: ModuleBuildTarget, dataManager: BuildDataManager): File = - File(dataManager.dataPaths.getTargetDataRoot(target), buildMetaInfoFileName) + fun isVersionChanged(chunk: KotlinChunk, buildMetaInfo: BuildMetaInfo): Boolean { + val file = chunk.buildMetaInfoFile(jpsModuleBuildTarget) + if (!file.exists()) return false - fun saveVersions(context: CompileContext, chunk: ModuleChunk, commonArguments: CommonCompilerArguments) { - val dataManager = context.projectDescriptor.dataManager - val targets = chunk.targets - val cacheVersionsProvider = CacheVersionProvider(dataManager.dataPaths, isIncrementalCompilationEnabled) - cacheVersionsProvider.allVersions(targets).forEach { it.saveIfNeeded() } - - val buildMetaInfo = buildMetaInfoFactory.create(commonArguments) - val serializedMetaInfo = buildMetaInfoFactory.serializeToString(buildMetaInfo) - - for (target in chunk.targets) { - buildMetaInfoFile(target, dataManager).writeText(serializedMetaInfo) - } - } - - fun checkCachesVersions(chunk: ModuleChunk, dataManager: BuildDataManager, actions: MutableSet) { - val args = compilerArgumentsForChunk(chunk) - val currentBuildMetaInfo = buildMetaInfoFactory.create(args) - - for (target in chunk.targets) { - val file = buildMetaInfoFile(target, dataManager) - if (!file.exists()) continue - - val lastBuildMetaInfo = - try { - buildMetaInfoFactory.deserializeFromString(file.readText()) ?: continue - } catch (e: Exception) { - KotlinBuilder.LOG.error("Could not deserialize build meta info", e) - continue - } - - val lastBuildLangVersion = LanguageVersion.fromVersionString(lastBuildMetaInfo.languageVersionString) - val lastBuildApiVersion = ApiVersion.parse(lastBuildMetaInfo.apiVersionString) - val currentLangVersion = - args.languageVersion?.let { LanguageVersion.fromVersionString(it) } ?: VersionView.RELEASED_VERSION - val currentApiVersion = - args.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.createByLanguageVersion(currentLangVersion) - - val reasonToRebuild = when { - currentLangVersion != lastBuildLangVersion -> { - "Language version was changed ($lastBuildLangVersion -> $currentLangVersion)" - } - - currentApiVersion != lastBuildApiVersion -> { - "Api version was changed ($lastBuildApiVersion -> $currentApiVersion)" - } - - lastBuildLangVersion != LanguageVersion.KOTLIN_1_0 && lastBuildMetaInfo.isEAP && !currentBuildMetaInfo.isEAP -> { - // If EAP->Non-EAP build with IC, then rebuild all kotlin - "Last build was compiled with EAP-plugin" - } - else -> null + val prevBuildMetaInfo = + try { + buildMetaInfoFactory.deserializeFromString(file.readText()) ?: return false + } catch (e: Exception) { + KotlinBuilder.LOG.error("Could not deserialize build meta info", e) + return false } - if (reasonToRebuild != null) { - KotlinBuilder.LOG.info("$reasonToRebuild. Performing non-incremental rebuild (kotlin only)") - actions.add(CacheVersion.Action.REBUILD_ALL_KOTLIN) + val prevLangVersion = LanguageVersion.fromVersionString(prevBuildMetaInfo.languageVersionString) + val prevApiVersion = ApiVersion.parse(prevBuildMetaInfo.apiVersionString) + + val reasonToRebuild = when { + chunk.langVersion != prevLangVersion -> "Language version was changed ($prevLangVersion -> ${chunk.langVersion})" + chunk.apiVersion != prevApiVersion -> "Api version was changed ($prevApiVersion -> ${chunk.apiVersion})" + prevLangVersion != LanguageVersion.KOTLIN_1_0 && prevBuildMetaInfo.isEAP && !buildMetaInfo.isEAP -> { + // If EAP->Non-EAP build with IC, then rebuild all kotlin + "Last build was compiled with EAP-plugin" } + else -> null } + + if (reasonToRebuild != null) { + KotlinBuilder.LOG.info("$reasonToRebuild. Performing non-incremental rebuild (kotlin only)") + return true + } + + return false } -} + + private fun checkRepresentativeTarget(chunk: KotlinChunk) { + check(chunk.representativeTarget == this) + } + + private fun checkRepresentativeTarget(chunk: ModuleChunk) { + check(chunk.representativeTarget() == jpsModuleBuildTarget) + } + + private fun checkRepresentativeTarget(chunk: List>) { + check(chunk.first() == this) + } +} \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinTargetsIndex.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinTargetsIndex.kt new file mode 100644 index 00000000000..fe9771197b6 --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/targets/KotlinTargetsIndex.kt @@ -0,0 +1,178 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.targets + +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.jps.model.java.JpsJavaClasspathKind +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.library.JpsOrderRootType +import org.jetbrains.jps.util.JpsPathUtil +import org.jetbrains.kotlin.jps.build.KotlinBuilder +import org.jetbrains.kotlin.jps.build.KotlinChunk +import org.jetbrains.kotlin.jps.build.KotlinCompileContext +import org.jetbrains.kotlin.jps.build.ModuleBuildTarget +import org.jetbrains.kotlin.jps.model.platform +import org.jetbrains.kotlin.platform.DefaultIdeTargetPlatformKindProvider +import org.jetbrains.kotlin.platform.IdePlatformKind +import org.jetbrains.kotlin.platform.impl.* +import org.jetbrains.kotlin.utils.LibraryUtils +import kotlin.system.measureTimeMillis + +class KotlinTargetsIndex( + val byJpsTarget: Map>, + val chunks: List, + val chunksByJpsRepresentativeTarget: Map +) + +internal class KotlinTargetsIndexBuilder internal constructor( + private val uninitializedContext: KotlinCompileContext +) { + private val byJpsModuleBuildTarget = mutableMapOf>() + private val isKotlinJsStdlibJar = mutableMapOf() + private val chunks = mutableListOf() + + fun build(): KotlinTargetsIndex { + val time = measureTimeMillis { + val jpsContext = uninitializedContext.jpsContext + + // visit all kotlin build targets + jpsContext.projectDescriptor.buildTargetIndex.getSortedTargetChunks(jpsContext).forEach { chunk -> + val moduleBuildTargets = chunk.targets.mapNotNull { + if (it is ModuleBuildTarget) ensureLoaded(it)!! + else null + } + + if (moduleBuildTargets.isNotEmpty()) { + val kotlinChunk = KotlinChunk(uninitializedContext, moduleBuildTargets) + moduleBuildTargets.forEach { + it.chunk = kotlinChunk + } + + chunks.add(kotlinChunk) + } + } + + calculateChunkDependencies() + } + + KotlinBuilder.LOG.info("KotlinTargetsIndex created in $time ms") + + return KotlinTargetsIndex( + byJpsModuleBuildTarget, + chunks, + chunks.associateBy { it.representativeTarget.jpsModuleBuildTarget } + ) + } + + private fun calculateChunkDependencies() { + chunks.forEach { chunk -> + val dependencies = mutableSetOf() + + chunk.targets.forEach { + dependencies.addAll(calculateTargetDependencies(it)) + } + + chunk.dependencies = dependencies.toList() + chunk.dependencies.forEach { dependency -> + dependency.target.chunk._dependent!!.add(dependency) + } + } + + chunks.forEach { + it.dependent = it._dependent!!.toList() + it._dependent = null + } + } + + private fun calculateTargetDependencies(srcTarget: KotlinModuleBuildTarget<*>): List { + val dependencies = mutableListOf() + val classpathKind = JpsJavaClasspathKind.compile(srcTarget.isTests) + + // TODO(1.2.80): Ask for JPS API + // Unfortunately JPS has no API for accessing "exported" flag while enumerating module dependencies, + // but has API for getting all and exported only dependent modules. + // So, lets first get set of all dependent targets, then remove exported only. + val dependentTargets = mutableSetOf>() + + JpsJavaExtensionService.dependencies(srcTarget.module) + .includedIn(classpathKind) + .processModules { destModule -> + val destKotlinTarget = byJpsModuleBuildTarget[ModuleBuildTarget(destModule, srcTarget.isTests)] + if (destKotlinTarget != null) { + dependentTargets.add(destKotlinTarget) + } + } + + JpsJavaExtensionService.dependencies(srcTarget.module) + .includedIn(classpathKind) + .exportedOnly() + .processModules { module -> + val destKotlinTarget = byJpsModuleBuildTarget[ModuleBuildTarget(module, srcTarget.isTests)] + if (destKotlinTarget != null) { + dependentTargets.remove(destKotlinTarget) + dependencies.add(KotlinModuleBuildTarget.Dependency(srcTarget, destKotlinTarget, true)) + } + } + + dependentTargets.forEach { destTarget -> + dependencies.add(KotlinModuleBuildTarget.Dependency(srcTarget, destTarget, false)) + } + + if (srcTarget.isTests) { + val srcProductionTarget = byJpsModuleBuildTarget[ModuleBuildTarget(srcTarget.module, false)] + if (srcProductionTarget != null) { + dependencies.add(KotlinModuleBuildTarget.Dependency(srcTarget, srcProductionTarget, true)) + } + } + + return dependencies + } + + + private fun ensureLoaded(target: ModuleBuildTarget): KotlinModuleBuildTarget<*>? { + return byJpsModuleBuildTarget.computeIfAbsent(target) { + val platform = target.module.platform?.kind ?: detectTargetPlatform(target) + + when { + platform.isCommon -> KotlinCommonModuleBuildTarget(uninitializedContext, target) + platform.isJavaScript -> KotlinJsModuleBuildTarget(uninitializedContext, target) + platform.isJvm -> KotlinJvmModuleBuildTarget(uninitializedContext, target) + else -> error("Unsupported platform $platform") + } + } + } + + /** + * Compatibility for KT-14082 + * todo: remove when all projects migrated to facets + */ + private fun detectTargetPlatform(target: ModuleBuildTarget): IdePlatformKind<*> { + if (hasJsStdLib(target)) return JsIdePlatformKind + + return DefaultIdeTargetPlatformKindProvider.defaultPlatform.kind + } + + private fun hasJsStdLib(target: ModuleBuildTarget): Boolean { + JpsJavaExtensionService.dependencies(target.module) + .recursively() + .exportedOnly() + .includedIn(JpsJavaClasspathKind.compile(target.isTests)) + .libraries + .forEach { library -> + for (root in library.getRoots(JpsOrderRootType.COMPILED)) { + val url = root.url + + val isKotlinJsLib = isKotlinJsStdlibJar.computeIfAbsent(url) { + LibraryUtils.isKotlinJavascriptStdLibrary(JpsPathUtil.urlToFile(url)) + } + + if (isKotlinJsLib) return true + } + } + + return false + } +} \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/build.log index c4525673afa..67d7e33ce4c 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/build.log @@ -1,7 +1,9 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/A.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/A.class @@ -12,7 +14,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/B.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -23,7 +27,9 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module3' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module3/src/C.kt Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/c/C.class @@ -34,7 +40,9 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module4' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module4/src/D.kt Cleaning output files: out/production/module4/D/D.class out/production/module4/META-INF/module4.kotlin_module @@ -43,4 +51,4 @@ Compiling files: module4/src/D.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/data-container-version-build.log index a0532d92bd3..c68c0096c7a 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/exportedModule/data-container-version-build.log @@ -1,7 +1,12 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/A.kt + module2/src/B.kt + module3/src/C.kt + module4/src/D.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/A.class @@ -9,15 +14,9 @@ End of files Compiling files: module1/src/A.kt End of files -Marked as dirty by Kotlin: - module1/src/A.kt - module2/src/B.kt - module3/src/C.kt - module4/src/D.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -28,7 +27,6 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/c/C.class @@ -39,7 +37,6 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module4/D/D.class out/production/module4/META-INF/module4.kotlin_module @@ -48,4 +45,4 @@ Compiling files: module4/src/D.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log index 147a909f94a..9d3d93f00fd 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/build.log @@ -1,7 +1,9 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/a.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -13,7 +15,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/b.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -24,7 +28,9 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module3' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module3/src/c.kt Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/module3/CKt.class @@ -35,7 +41,9 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module4' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module4/src/d.kt Cleaning output files: out/production/module4/META-INF/module4.kotlin_module out/production/module4/module4/D.class @@ -47,4 +55,4 @@ Exit code: OK ------------------------------------------ Building module5 Exit code: NOTHING_DONE ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log index 4a55c7fdaf2..894814e7db8 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module1Modified/data-container-version-build.log @@ -1,7 +1,12 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/a.kt + module2/src/b.kt + module3/src/c.kt + module4/src/d.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -10,15 +15,9 @@ End of files Compiling files: module1/src/a.kt End of files -Marked as dirty by Kotlin: - module1/src/a.kt - module2/src/b.kt - module3/src/c.kt - module4/src/d.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -29,7 +28,6 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/module3/CKt.class @@ -40,7 +38,6 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module4/META-INF/module4.kotlin_module out/production/module4/module4/D.class @@ -52,4 +49,4 @@ Exit code: OK ------------------------------------------ Building module5 Exit code: NOTHING_DONE ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/build.log index 387f12ffca8..20c57b2a0fa 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/build.log @@ -1,7 +1,9 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/a.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -13,7 +15,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/b.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -22,4 +26,4 @@ Compiling files: module2/src/b.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/data-container-version-build.log index 0c32827933a..570bf2a3fc1 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/module2Modified/data-container-version-build.log @@ -1,7 +1,10 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/a.kt + module2/src/b.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -10,13 +13,9 @@ End of files Compiling files: module1/src/a.kt End of files -Marked as dirty by Kotlin: - module1/src/a.kt - module2/src/b.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -25,4 +24,4 @@ Compiling files: module2/src/b.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log index 46c3182b0b3..bafd696ae08 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/build.log @@ -1,7 +1,9 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/A.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/AKt.class @@ -12,7 +14,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/B.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -21,4 +25,4 @@ Compiling files: module2/src/B.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/data-container-version-build.log index bfa894afa17..74345f4d7d2 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithConstantModified/data-container-version-build.log @@ -1,7 +1,10 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/A.kt + module2/src/B.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/AKt.class @@ -9,13 +12,9 @@ End of files Compiling files: module1/src/A.kt End of files -Marked as dirty by Kotlin: - module1/src/A.kt - module2/src/B.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -24,4 +23,4 @@ Compiling files: module2/src/B.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log index cdd46868ec5..edfc73b0c2e 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/build.log @@ -1,7 +1,9 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/A.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/A.class @@ -13,7 +15,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/B.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -22,4 +26,4 @@ Compiling files: module2/src/B.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/data-container-version-build.log index 3eb02e0f67a..2296ff45e59 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/moduleWithInlineModified/data-container-version-build.log @@ -1,7 +1,10 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/A.kt + module2/src/B.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/a/A.class @@ -10,13 +13,9 @@ End of files Compiling files: module1/src/A.kt End of files -Marked as dirty by Kotlin: - module1/src/A.kt - module2/src/B.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/b/B.class @@ -25,4 +24,4 @@ Compiling files: module2/src/B.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/build.log index 781c9ca61d2..74777f27218 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/build.log @@ -1,6 +1,10 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt + src/other.kt Cleaning output files: out/production/module/META-INF/module.kotlin_module out/production/module/other/OtherKt.class diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/data-container-version-build.log index 9f939ce73a2..50ccb3eaacf 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/touchedFile/data-container-version-build.log @@ -1,6 +1,10 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt + src/other.kt Cleaning output files: out/production/module/META-INF/module.kotlin_module out/production/module/other/OtherKt.class @@ -12,9 +16,5 @@ Compiling files: src/b.kt src/other.kt End of files -Marked as dirty by Kotlin: - src/a.kt - src/b.kt - src/other.kt Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/build.log index c48fd6f8bbc..40e0cec5d0a 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/build.log @@ -1,6 +1,10 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt + src/other.kt Cleaning output files: out/production/module/A.class out/production/module/META-INF/module.kotlin_module @@ -17,4 +21,4 @@ Exit code: OK ------------------------------------------ Compiling files: src/A.java -End of files +End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/data-container-version-build.log index 2a22b77369b..6969fef9b7f 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/touchedOnlyJavaFile/data-container-version-build.log @@ -1,6 +1,10 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt + src/other.kt Cleaning output files: out/production/module/A.class out/production/module/META-INF/module.kotlin_module @@ -13,12 +17,8 @@ Compiling files: src/b.kt src/other.kt End of files -Marked as dirty by Kotlin: - src/a.kt - src/b.kt - src/other.kt Exit code: OK ------------------------------------------ Compiling files: src/A.java -End of files +End of files \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/build.log index 961547cb08d..dc9b46c066a 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/build.log @@ -1,6 +1,9 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt Cleaning output files: out/production/module/META-INF/module.kotlin_module out/production/module/test/AKt.class @@ -11,4 +14,4 @@ Compiling files: src/b.kt End of files Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/data-container-version-build.log index 280d11f5d43..5e1e9069b1c 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/untouchedFiles/data-container-version-build.log @@ -1,6 +1,9 @@ ================ Step #1 ================= -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + src/a.kt + src/b.kt Cleaning output files: out/production/module/META-INF/module.kotlin_module out/production/module/test/AKt.class @@ -10,8 +13,5 @@ Compiling files: src/a.kt src/b.kt End of files -Marked as dirty by Kotlin: - src/a.kt - src/b.kt Exit code: OK ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/withError/build.log b/jps-plugin/testData/incremental/cacheVersionChanged/withError/build.log index f3c27df32a1..c8c238cd23f 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/withError/build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/withError/build.log @@ -1,7 +1,10 @@ ================ Step #1 ================= Building module1 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module1' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module1/src/a.kt + module1/src/f.kt Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -27,7 +30,9 @@ End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module2' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module2/src/b.kt Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -38,7 +43,9 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module3' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module3/src/c.kt Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/module3/CKt.class @@ -49,7 +56,9 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] +Local cache for KotlinChunk(Module 'module4' production) are INVALID: actual=CacheVersion(version=777) -> expected=CacheVersion(version=9011001) +Marked as dirty by Kotlin: + module4/src/d.kt Cleaning output files: out/production/module4/META-INF/module4.kotlin_module out/production/module4/module4/D.class @@ -61,4 +70,4 @@ Exit code: OK ------------------------------------------ Building module5 Exit code: NOTHING_DONE ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/cacheVersionChanged/withError/data-container-version-build.log b/jps-plugin/testData/incremental/cacheVersionChanged/withError/data-container-version-build.log index d2337355e89..6d1a9032e85 100644 --- a/jps-plugin/testData/incremental/cacheVersionChanged/withError/data-container-version-build.log +++ b/jps-plugin/testData/incremental/cacheVersionChanged/withError/data-container-version-build.log @@ -1,7 +1,13 @@ ================ Step #1 ================= +Lookups cache are INVALID: actual=(777, [jvm]) -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/a.kt + module1/src/f.kt + module2/src/b.kt + module3/src/c.kt + module4/src/d.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, DO_NOTHING] Cleaning output files: out/production/module1/META-INF/module1.kotlin_module out/production/module1/module1/A.class @@ -12,12 +18,6 @@ Compiling files: module1/src/a.kt module1/src/f.kt End of files -Marked as dirty by Kotlin: - module1/src/a.kt - module1/src/f.kt - module2/src/b.kt - module3/src/c.kt - module4/src/d.kt Exit code: ABORT ------------------------------------------ COMPILATION FAILED @@ -26,21 +26,13 @@ Name expected ================ Step #2 ================= Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, REBUILD_CHUNK] Compiling files: module1/src/a.kt module1/src/f.kt End of files -Marked as dirty by Kotlin: - module1/src/a.kt - module1/src/f.kt - module2/src/b.kt - module3/src/c.kt - module4/src/d.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/module2/BKt.class @@ -51,7 +43,6 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/module3/CKt.class @@ -62,7 +53,6 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module4/META-INF/module4.kotlin_module out/production/module4/module4/D.class @@ -74,4 +64,4 @@ Exit code: OK ------------------------------------------ Building module5 Exit code: NOTHING_DONE ------------------------------------------- +------------------------------------------ \ No newline at end of file diff --git a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOff/build.log b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOff/build.log index 337c3ad1c41..0fecab40dc7 100644 --- a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOff/build.log +++ b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOff/build.log @@ -1,7 +1,11 @@ ================ Step #1 ================= +Lookups cache are SHOULD_BE_CLEARED: actual=(3011001, [jvm]) -> expected=null +Local cache for Module 'module1' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module2' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module3' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module4' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null Building module1 -Actions after cache changed: [CLEAN_NORMAL_CACHES, CLEAN_DATA_CONTAINER] Exit code: NOTHING_DONE ------------------------------------------ Building module2 diff --git a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log index 2067766ef1a..2ab5851d132 100644 --- a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log +++ b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOn/build.log @@ -1,7 +1,11 @@ ================ Step #1 ================= +Lookups cache are SHOULD_BE_CLEARED: actual=(3011001, [jvm]) -> expected=null +Local cache for Module 'module1' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module2' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module3' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module4' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null Building module1 -Actions after cache changed: [CLEAN_NORMAL_CACHES, CLEAN_DATA_CONTAINER] Exit code: NOTHING_DONE ------------------------------------------ Building module2 @@ -31,15 +35,7 @@ Exit code: NOTHING_DONE ================ Step #2 ================= -Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, REBUILD_CHUNK] -Cleaning output files: - out/production/module1/META-INF/module1.kotlin_module - out/production/module1/foo/Z.class -End of files -Compiling files: - module1/src/z.kt -End of files +Lookups cache are INVALID: actual=null -> expected=(3011001, [jvm]) Marked as dirty by Kotlin: module1/src/z.kt module2/src/a.kt @@ -47,10 +43,17 @@ Marked as dirty by Kotlin: module2/src/c.kt module3/src/d.kt module4/src/e.kt +Building module1 +Cleaning output files: + out/production/module1/META-INF/module1.kotlin_module + out/production/module1/foo/Z.class +End of files +Compiling files: + module1/src/z.kt +End of files Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/META-INF/module2.kotlin_module out/production/module2/foo/AKt.class @@ -65,7 +68,6 @@ End of files Exit code: OK ------------------------------------------ Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module3/META-INF/module3.kotlin_module out/production/module3/foo/D.class @@ -76,7 +78,6 @@ End of files Exit code: OK ------------------------------------------ Building module4 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module4/META-INF/module4.kotlin_module out/production/module4/foo/E.class diff --git a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOnJavaChanged/build.log b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOnJavaChanged/build.log index eae4e20316b..ddafb9748d7 100644 --- a/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOnJavaChanged/build.log +++ b/jps-plugin/testData/incremental/changeIncrementalOption/incrementalOffOnJavaChanged/build.log @@ -1,7 +1,10 @@ ================ Step #1 ================= +Lookups cache are SHOULD_BE_CLEARED: actual=(3011001, [jvm]) -> expected=null +Local cache for Module 'module1' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module2' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null +Local cache for Module 'module3' production are SHOULD_BE_CLEARED: actual=CacheVersion(version=9011001) -> expected=null Building module1 -Actions after cache changed: [CLEAN_NORMAL_CACHES, CLEAN_DATA_CONTAINER] Exit code: NOTHING_DONE ------------------------------------------ Building module2 @@ -13,8 +16,12 @@ Exit code: NOTHING_DONE ================ Step #2 ================= +Lookups cache are INVALID: actual=null -> expected=(3011001, [jvm]) +Marked as dirty by Kotlin: + module1/src/a.kt + module2/src/c.kt + module3/src/d.kt Building module1 -Actions after cache changed: [REBUILD_ALL_KOTLIN, REBUILD_CHUNK] Cleaning output files: out/production/module1/AKt.class out/production/module1/META-INF/module1.kotlin_module @@ -22,14 +29,9 @@ End of files Compiling files: module1/src/a.kt End of files -Marked as dirty by Kotlin: - module1/src/a.kt - module2/src/c.kt - module3/src/d.kt Exit code: OK ------------------------------------------ Building module2 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module2/B.class out/production/module2/CKt.class @@ -44,7 +46,6 @@ Compiling files: module2/src/B.java End of files Building module3 -Actions after cache changed: [REBUILD_CHUNK, DO_NOTHING] Cleaning output files: out/production/module3/DKt.class out/production/module3/META-INF/module3.kotlin_module diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/noKotlin/build.log b/jps-plugin/testData/incremental/lazyKotlinCaches/noKotlin/build.log index 4199fbdec4e..313a544ce5b 100644 --- a/jps-plugin/testData/incremental/lazyKotlinCaches/noKotlin/build.log +++ b/jps-plugin/testData/incremental/lazyKotlinCaches/noKotlin/build.log @@ -7,4 +7,4 @@ Exit code: NOTHING_DONE ------------------------------------------ Compiling files: src/A.java -End of files +End of files \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/gradleCacheVersion.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/gradleCacheVersion.kt index 1f2bd361d12..327ab5dc79f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/gradleCacheVersion.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/gradleCacheVersion.kt @@ -5,15 +5,15 @@ package org.jetbrains.kotlin.gradle.incremental -import org.jetbrains.kotlin.incremental.CacheVersion -import org.jetbrains.kotlin.incremental.customCacheVersion +import org.jetbrains.kotlin.incremental.customCacheVersionManager +import org.jetbrains.kotlin.incremental.storage.version.CacheVersionManager import java.io.File internal const val GRADLE_CACHE_VERSION = 4 internal const val GRADLE_CACHE_VERSION_FILE_NAME = "gradle-format-version.txt" -internal fun gradleCacheVersion(dataRoot: File, enabled: Boolean): CacheVersion = - customCacheVersion( +internal fun gradleCacheVersionManager(dataRoot: File, enabled: Boolean): CacheVersionManager = + customCacheVersionManager( GRADLE_CACHE_VERSION, GRADLE_CACHE_VERSION_FILE_NAME, dataRoot,