Enable separate versioning for experimental incremental compilation

This commit is contained in:
Alexey Tsvetkov
2015-11-16 15:57:24 +03:00
parent d20fb5ddd7
commit 69c906014f
5 changed files with 289 additions and 112 deletions
@@ -23,7 +23,7 @@ public class IncrementalCompilation {
private static final String IS_EXPERIMENTAL_PROPERTY = "kotlin.incremental.compilation.experimental";
public static boolean isExperimental() {
return "true".equals(System.getProperty(IS_EXPERIMENTAL_PROPERTY));
return isEnabled() && "true".equals(System.getProperty(IS_EXPERIMENTAL_PROPERTY));
}
public static boolean isEnabled() {
@@ -72,10 +72,26 @@ import java.util.*
public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
companion object {
// TODO add description to string
private val TARGETS_WITH_CLEARED_CACHES = Key<Set<ModuleBuildTarget>>("")
public val KOTLIN_BUILDER_NAME: String = "Kotlin Builder"
public val LOOKUP_TRACKER: JpsElementChildRoleBase<JpsSimpleElement<out LookupTracker>> = JpsElementChildRoleBase.create("lookup tracker")
val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
private fun registerTargetsWithClearedCaches(context: CompileContext, targets: Set<ModuleBuildTarget>) {
synchronized(TARGETS_WITH_CLEARED_CACHES) {
val data = (context.getUserData(TARGETS_WITH_CLEARED_CACHES) ?: setOf()) + targets
context.putUserData(TARGETS_WITH_CLEARED_CACHES, data)
}
}
private fun unregisterTargetsWithClearedCaches(context: CompileContext, targets: Set<ModuleBuildTarget>) {
synchronized(TARGETS_WITH_CLEARED_CACHES) {
val data = (context.getUserData(TARGETS_WITH_CLEARED_CACHES) ?: setOf()) - targets
context.putUserData(TARGETS_WITH_CLEARED_CACHES, data)
}
}
}
private val statisticsLogger = TeamcityStatisticsLogger()
@@ -106,9 +122,14 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
val messageCollector = MessageCollectorAdapter(context)
try {
val result = doBuild(chunk, context, dirtyFilesHolder, messageCollector, outputConsumer)
LOG.debug("Build result: " + result)
return result
val exitCode = doBuild(chunk, context, dirtyFilesHolder, messageCollector, outputConsumer)
LOG.debug("Build result: " + exitCode)
if (exitCode != ExitCode.CHUNK_REBUILD_REQUIRED && exitCode != ABORT) {
saveVersions(context, chunk)
}
return exitCode
}
catch (e: StopBuildException) {
LOG.debug("Caught exception: " + e)
@@ -139,17 +160,21 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
}
val projectDescriptor = context.projectDescriptor
val dataManager = projectDescriptor.dataManager
val targets = chunk.targets
val requestedToRebuild = context.getUserData(TARGETS_WITH_CLEARED_CACHES) ?: setOf()
val isFullRebuild = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)
if (chunk.targets.any { dataManager.dataPaths.getKotlinCacheVersion(it).isIncompatible() }) {
LOG.info("Clearing caches for " + chunk.targets.joinToString { it.presentableName })
chunk.targets.forEach { dataManager.getKotlinCache(it).clean() }
return CHUNK_REBUILD_REQUIRED
if (!isFullRebuild && !requestedToRebuild.containsAll(targets)) {
val exitCode = checkVersions(context, dataManager, targets)
if (exitCode != null) return exitCode
}
if (!dirtyFilesHolder.hasDirtyFiles() && !dirtyFilesHolder.hasRemovedFiles()
|| !hasKotlinDirtyOrRemovedFiles(dirtyFilesHolder, chunk)) {
if (!dirtyFilesHolder.hasDirtyFiles() &&
!dirtyFilesHolder.hasRemovedFiles() ||
!hasKotlinDirtyOrRemovedFiles(dirtyFilesHolder, chunk)
) {
return NOTHING_DONE
}
@@ -200,17 +225,22 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
return OK
}
val generatedClasses = generatedFiles.filterIsInstance<GeneratedJvmClass>()
updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, generatedClasses)
if (!IncrementalCompilation.isEnabled()) {
return OK
}
context.checkCanceled()
val generatedClasses = generatedFiles.filterIsInstance<GeneratedJvmClass>()
val changesInfo = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedFiles, chunk)
updateJavaMappings(chunk, compilationErrors, context, dirtyFilesHolder, filesToCompile, generatedClasses)
val changesInfo = updateKotlinIncrementalCache(compilationErrors, incrementalCaches, generatedFiles)
updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile)
if (isFullRebuild) {
return OK
}
val caches = filesToCompile.keySet().map { incrementalCaches[it]!! }
processChanges(context, chunk, filesToCompile.values(), allCompiledFiles, dataManager, caches, changesInfo)
@@ -300,6 +330,77 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
}
}
private fun checkVersions(context: CompileContext, dataManager: BuildDataManager, targets: MutableSet<ModuleBuildTarget>): ExitCode? {
val cacheVersionsProvider = CacheVersionProvider(dataManager.dataPaths)
val allVersions = cacheVersionsProvider.allVersions(targets)
val actions = allVersions.map { it.checkVersion() }.toSet().sorted()
fun cleanNormalCaches() {
LOG.info("Clearing caches for " + targets.joinToString { it.presentableName })
targets.forEach { dataManager.getKotlinCache(it).clean() }
}
for (status in actions) {
when (status) {
CacheVersion.Action.REBUILD_ALL_KOTLIN -> {
LOG.info("Kotlin global lookup map format changed, so rebuild all kotlin")
val project = context.projectDescriptor.project
val sourceRoots = project.modules.flatMap { it.sourceRoots }
for (sourceRoot in sourceRoots) {
val ktFiles = sourceRoot.file.walk().filter { KotlinSourceFileCollector.isKotlinSourceFile(it) }
ktFiles.forEach { kt ->
FSOperations.markDirty(context, CompilationRound.NEXT, kt)
}
}
val buildTargetIndex = context.projectDescriptor.buildTargetIndex
val allTargets = buildTargetIndex.allTargets.filterIsInstance<ModuleBuildTarget>().toSet()
for (target in targets) {
dataManager.getKotlinCache(target).clean()
}
dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider).clean()
registerTargetsWithClearedCaches(context, allTargets)
return CHUNK_REBUILD_REQUIRED
}
CacheVersion.Action.REBUILD_CHUNK -> {
cleanNormalCaches()
registerTargetsWithClearedCaches(context, targets)
return CHUNK_REBUILD_REQUIRED
}
CacheVersion.Action.CLEAN_NORMAL_CACHES -> {
cleanNormalCaches()
}
CacheVersion.Action.CLEAN_EXPERIMENTAL_CACHES -> {
LOG.info("Clearing experimental caches for " + targets.joinToString { it.presentableName })
targets.forEach { dataManager.getKotlinCache(it).cleanExperimental() }
}
CacheVersion.Action.CLEAN_DATA_CONTAINER -> {
LOG.info("Clearing lookup cache")
dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider).clean()
cacheVersionsProvider.dataContainerVersion().clean()
}
else -> {
assert(status == CacheVersion.Action.DO_NOTHING) { "Unknown version status $status" }
}
}
}
return null
}
private fun saveVersions(context: CompileContext, chunk: ModuleChunk) {
val dataManager = context.projectDescriptor.dataManager
val targets = chunk.targets
val cacheVersionsProvider = CacheVersionProvider(dataManager.dataPaths)
cacheVersionsProvider.allVersions(targets).forEach { it.saveIfNeeded() }
unregisterTargetsWithClearedCaches(context, targets)
}
private fun doCompileModuleChunk(
allCompiledFiles: MutableSet<File>, chunk: ModuleChunk, commonArguments: CommonCompilerArguments, context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>, environment: CompilerEnvironment,
@@ -414,8 +515,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
filesToCompile: MultiMap<ModuleBuildTarget, File>,
generatedClasses: List<GeneratedJvmClass>
) {
assert(IncrementalCompilation.isEnabled()) { "updateJavaMappings should not be called when incremental compilation disabled" }
val previousMappings = context.getProjectDescriptor().dataManager.getMappings()
val delta = previousMappings.createDelta()
val callback = delta.getCallback()
@@ -442,14 +541,11 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
private fun updateKotlinIncrementalCache(
compilationErrors: Boolean,
incrementalCaches: Map<ModuleBuildTarget, IncrementalCacheImpl>,
generatedFiles: List<GeneratedFile>,
chunk: ModuleChunk
generatedFiles: List<GeneratedFile>
): CompilationResult {
assert(IncrementalCompilation.isEnabled()) { "updateKotlinIncrementalCache should not be called when incremental compilation disabled" }
chunk.targets.forEach { incrementalCaches[it]!!.saveCacheFormatVersion() }
var changesInfo = CompilationResult.NO_CHANGES
for (generatedFile in generatedFiles) {
val ic = incrementalCaches[generatedFile.target]!!
@@ -704,6 +800,7 @@ private fun getIncrementalCaches(chunk: ModuleChunk, context: CompileContext): M
return caches
}
// TODO: investigate thread safety
private val ALL_COMPILED_FILES_KEY = Key.create<MutableSet<File>>("_all_kotlin_compiled_files_")
private fun getAllCompiledFilesContainer(context: CompileContext): MutableSet<File> {
var allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context)
@@ -714,6 +811,7 @@ private fun getAllCompiledFilesContainer(context: CompileContext): MutableSet<Fi
return allCompiledFiles
}
// TODO: investigate thread safety
private val PROCESSED_TARGETS_WITH_REMOVED_FILES = Key.create<MutableSet<ModuleBuildTarget>>("_processed_targets_with_removed_files_")
private fun getProcessedTargetsWithRemovedFilesContainer(context: CompileContext): MutableSet<ModuleBuildTarget> {
var set = PROCESSED_TARGETS_WITH_REMOVED_FILES.get(context)
@@ -1,70 +0,0 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.load.java.JvmAbi
import java.io.File
class CacheFormatVersion(targetDataRoot: File) {
companion object {
// Change this when incremental cache format changes
private val INCREMENTAL_CACHE_OWN_VERSION = 7
private val CACHE_FORMAT_VERSION =
INCREMENTAL_CACHE_OWN_VERSION * 1000000 +
JvmAbi.VERSION.major * 1000 +
JvmAbi.VERSION.minor
private val FORMAT_VERSION_FILE_PATH: String = "$CACHE_DIRECTORY_NAME/format-version.txt"
}
private val file = File(targetDataRoot, FORMAT_VERSION_FILE_PATH)
fun isIncompatible(): Boolean {
if (!file.exists()) return false
if (!IncrementalCompilation.isEnabled()) return true
val versionNumber = file.readText().toInt()
if (versionNumber != CACHE_FORMAT_VERSION) {
KotlinBuilder.LOG.info("Incompatible incremental cache version, expected $CACHE_FORMAT_VERSION, actual $versionNumber")
return true
}
return false
}
fun saveIfNeeded() {
if (!file.parentFile.exists()) {
file.parentFile.mkdirs()
}
file.writeText("$CACHE_FORMAT_VERSION")
}
fun clean() {
file.delete()
}
@TestOnly
val formatVersionFile: File
get() = file
}
@@ -0,0 +1,155 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental
import org.jetbrains.annotations.TestOnly
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.load.java.JvmAbi
import java.io.File
private val NORMAL_VERSION = 7
private val EXPERIMENTAL_VERSION = 1
private val DATA_CONTAINER_VERSION = 1
private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
private val EXPERIMENTAL_VERSION_FILE_NAME = "experimental-format-version.txt"
private val DATA_CONTAINER_VERSION_FILE_NAME = "data-container-format-version.txt"
sealed class CacheVersion {
protected abstract val ownVersion: Int
protected abstract val versionFile: File
protected abstract val isEnabled: Boolean
protected abstract val whenVersionChanged: CacheVersion.Action
protected abstract val whenTurnedOn: CacheVersion.Action
protected abstract val whenTurnedOff: CacheVersion.Action
private val actualVersion: Int
get() = versionFile.readText().toInt()
private val expectedVersion: Int
get() = ownVersion * 1000000 + JvmAbi.VERSION.major * 1000 + JvmAbi.VERSION.minor
fun checkVersion(): Action =
when (versionFile.exists() to isEnabled) {
true to true -> if (actualVersion != expectedVersion) whenVersionChanged else Action.DO_NOTHING
false to true -> whenTurnedOn
true to false -> whenTurnedOff
else -> Action.DO_NOTHING
}
fun saveIfNeeded() {
if (!isEnabled) return
if (!versionFile.parentFile.exists()) {
versionFile.parentFile.mkdirs()
}
versionFile.writeText(expectedVersion.toString())
}
fun clean() {
versionFile.delete()
}
@TestOnly
val formatVersionFile: File
get() = versionFile
// Order of entries is important, because actions are sorted in KotlinBuilder::checkVersions
enum class Action {
REBUILD_ALL_KOTLIN,
REBUILD_CHUNK,
CLEAN_NORMAL_CACHES,
CLEAN_EXPERIMENTAL_CACHES,
CLEAN_DATA_CONTAINER,
DO_NOTHING
}
class Normal(dir: File) : CacheVersion() {
override val ownVersion = NORMAL_VERSION
override val versionFile = File(dir, NORMAL_VERSION_FILE_NAME)
override val isEnabled: Boolean
get() = IncrementalCompilation.isEnabled()
override val whenVersionChanged = Action.REBUILD_CHUNK
override val whenTurnedOn = Action.REBUILD_CHUNK
override val whenTurnedOff = Action.CLEAN_NORMAL_CACHES
}
class Experimental(dir: File) : CacheVersion() {
override val ownVersion = EXPERIMENTAL_VERSION
override val versionFile = File(dir, EXPERIMENTAL_VERSION_FILE_NAME)
override val isEnabled: Boolean
get() = IncrementalCompilation.isExperimental()
override val whenVersionChanged = Action.REBUILD_CHUNK
override val whenTurnedOn = Action.REBUILD_CHUNK
override val whenTurnedOff = Action.CLEAN_EXPERIMENTAL_CACHES
}
class DataContainer(dir: File) : CacheVersion() {
override val ownVersion = DATA_CONTAINER_VERSION
override val versionFile = File(dir, DATA_CONTAINER_VERSION_FILE_NAME)
override val isEnabled: Boolean
get() = IncrementalCompilation.isExperimental()
override val whenVersionChanged = Action.REBUILD_ALL_KOTLIN
override val whenTurnedOn = Action.REBUILD_ALL_KOTLIN
override val whenTurnedOff = Action.CLEAN_DATA_CONTAINER
}
}
class CacheVersionProvider(private val paths: BuildDataPaths) {
protected val BuildTarget<*>.dataRoot: File
get() = paths.getTargetDataRoot(this)
fun normalVersion(target: ModuleBuildTarget): CacheVersion =
CacheVersion.Normal(target.dataRoot)
fun experimentalVersion(target: ModuleBuildTarget): CacheVersion =
CacheVersion.Experimental(target.dataRoot)
fun dataContainerVersion(): CacheVersion =
CacheVersion.DataContainer(KotlinDataContainerTarget.dataRoot)
fun allVersions(targets: Iterable<ModuleBuildTarget>): Iterable<CacheVersion> {
val versions = arrayListOf<CacheVersion>()
versions.add(dataContainerVersion())
for (target in targets) {
versions.add(normalVersion(target))
versions.add(experimentalVersion(target))
}
return versions
}
}
@@ -24,7 +24,6 @@ import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.IOUtil
import gnu.trove.THashSet
import org.jetbrains.annotations.TestOnly
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.storage.BuildDataPaths
import org.jetbrains.jps.builders.storage.StorageProvider
import org.jetbrains.jps.incremental.ModuleBuildTarget
@@ -55,13 +54,9 @@ import java.util.*
internal val CACHE_DIRECTORY_NAME = "kotlin"
@TestOnly
public fun getCacheDirectoryName(): String =
CACHE_DIRECTORY_NAME
public class IncrementalCacheImpl(
targetDataRoot: File,
private val target: ModuleBuildTarget
private val target: ModuleBuildTarget,
paths: BuildDataPaths
) : BasicMapsOwner(), IncrementalCache {
companion object {
val PROTO_MAP = "proto"
@@ -78,7 +73,8 @@ public class IncrementalCacheImpl(
private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
}
private val baseDir = File(targetDataRoot, CACHE_DIRECTORY_NAME)
private val baseDir = File(paths.getTargetDataRoot(target), CACHE_DIRECTORY_NAME)
private val cacheVersionProvider = CacheVersionProvider(paths)
private val String.storageFile: File
get() = File(baseDir, this + "." + CACHE_EXTENSION)
@@ -94,7 +90,6 @@ public class IncrementalCacheImpl(
private val dirtyInlineFunctionsMap = registerMap(DirtyInlineFunctionsMap(DIRTY_INLINE_FUNCTIONS.storageFile))
private val inlinedTo = registerMap(InlineFunctionsFilesMap(INLINED_TO.storageFile))
private val cacheFormatVersion = CacheFormatVersion(targetDataRoot)
private val dependents = arrayListOf<IncrementalCacheImpl>()
private val outputDir = requireNotNull(target.outputDir) { "Target is expected to have output directory: $target" }
@@ -144,10 +139,6 @@ public class IncrementalCacheImpl(
return toSystemIndependentName(File(outputDir, "$internalClassName.class").canonicalPath)
}
public fun saveCacheFormatVersion() {
cacheFormatVersion.saveIfNeeded()
}
public fun saveModuleMappingToCache(sourceFiles: Collection<File>, file: File): CompilationResult {
val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)
protoMap.process(jvmClassName, file.readBytes(), emptyArray<String>(), isPackage = false, checkChangesIsOpenPart = false)
@@ -320,7 +311,12 @@ public class IncrementalCacheImpl(
public override fun clean() {
super.clean()
cacheFormatVersion.clean()
cacheVersionProvider.normalVersion(target).clean()
cacheVersionProvider.experimentalVersion(target).clean()
}
public fun cleanExperimental() {
cacheVersionProvider.experimentalVersion(target).clean()
}
private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
@@ -677,11 +673,9 @@ data class CompilationResult(
changes + other.changes)
}
public fun BuildDataPaths.getKotlinCacheVersion(target: BuildTarget<*>): CacheFormatVersion = CacheFormatVersion(getTargetDataRoot(target))
private class KotlinIncrementalStorageProvider(
private val target: ModuleBuildTarget
private val target: ModuleBuildTarget,
private val paths: BuildDataPaths
) : StorageProvider<IncrementalCacheImpl>() {
override fun equals(other: Any?) = other is KotlinIncrementalStorageProvider && target == other.target
@@ -689,11 +683,11 @@ private class KotlinIncrementalStorageProvider(
override fun hashCode() = target.hashCode()
override fun createStorage(targetDataDir: File): IncrementalCacheImpl =
IncrementalCacheImpl(targetDataDir, target)
IncrementalCacheImpl(target, paths)
}
public fun BuildDataManager.getKotlinCache(target: ModuleBuildTarget): IncrementalCacheImpl =
getStorage(target, KotlinIncrementalStorageProvider(target))
fun BuildDataManager.getKotlinCache(target: ModuleBuildTarget): IncrementalCacheImpl =
getStorage(target, KotlinIncrementalStorageProvider(target, dataPaths))
private fun ByteArray.md5(): Long {
val d = MessageDigest.getInstance("MD5").digest(this)!!