diff --git a/jps/jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider b/jps/jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider deleted file mode 100644 index eadf409aee9..00000000000 --- a/jps/jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider +++ /dev/null @@ -1 +0,0 @@ -org.jetbrains.jet.jps.incremental.IncrementalCacheProviderImpl \ No newline at end of file diff --git a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java index a91685071d8..91a4814c41e 100644 --- a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java +++ b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java @@ -16,6 +16,7 @@ package org.jetbrains.jet.jps.build; +import com.google.common.collect.Maps; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.StreamUtil; @@ -24,7 +25,6 @@ import com.intellij.util.Function; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.MultiMap; import gnu.trove.THashSet; -import kotlin.Pair; import org.jetbrains.annotations.NotNull; import org.jetbrains.jet.cli.common.KotlinVersion; import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments; @@ -40,17 +40,17 @@ import org.jetbrains.jet.compiler.runner.OutputItemsCollectorImpl; import org.jetbrains.jet.compiler.runner.SimpleOutputItem; import org.jetbrains.jet.config.IncrementalCompilation; import org.jetbrains.jet.jps.JpsKotlinCompilerSettings; -import org.jetbrains.jet.jps.incremental.IncrementalCacheImpl; +import org.jetbrains.jet.jps.incremental.*; import org.jetbrains.jet.preloading.ClassLoaderFactory; import org.jetbrains.jet.utils.PathUtil; import org.jetbrains.jps.ModuleChunk; -import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.DirtyFilesHolder; import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor; import org.jetbrains.jps.incremental.*; import org.jetbrains.jps.incremental.java.JavaBuilder; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; +import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.model.JpsProject; import org.jetbrains.jps.model.module.JpsModule; @@ -123,13 +123,22 @@ public class KotlinBuilder extends ModuleLevelBuilder { File outputDir = representativeTarget.getOutputDir(); + BuildDataManager dataManager = context.getProjectDescriptor().dataManager; + Map incrementalCaches = Maps.newHashMap(); + for (ModuleBuildTarget target : chunk.getTargets()) { + incrementalCaches.put(target, dataManager.getStorage(target, IncrementalCacheStorageProvider.INSTANCE$)); + } + + IncrementalCacheProviderImpl cacheProvider = new IncrementalCacheProviderImpl(incrementalCaches); + CompilerEnvironment environment = CompilerEnvironment.getEnvironmentFor( PathUtil.getKotlinPathsForJpsPluginOrJpsTests(), outputDir, new ClassLoaderFactory() { @Override public ClassLoader create(ClassLoader compilerClassLoader) { return new MyClassLoader(compilerClassLoader); } - } + }, + cacheProvider ); if (!environment.success()) { environment.reportErrorsTo(messageCollector); @@ -213,7 +222,7 @@ public class KotlinBuilder extends ModuleLevelBuilder { } // If there's only one target, this map is empty: get() always returns null, and the representativeTarget will be used below - Map> sourceToTarget = new HashMap>(); + Map sourceToTarget = new HashMap(); if (chunk.getTargets().size() > 1) { for (ModuleBuildTarget target : chunk.getTargets()) { for (File file : KotlinSourceFileCollector.getAllKotlinSourceFiles(target)) { @@ -222,67 +231,62 @@ public class KotlinBuilder extends ModuleLevelBuilder { } } - IncrementalCacheImpl cache = new IncrementalCacheImpl(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context)); + for (ModuleBuildTarget target : chunk.getTargets()) { + String targetId = target.getId(); - try { - List> moduleIdsAndSourceFiles = new ArrayList>(); - Map outDirectories = new HashMap(); + List removedFiles = ContainerUtil.map(KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target), + new Function() { + @Override + public File fun(String s) { + return new File(s); + } + }); - for (ModuleBuildTarget target : chunk.getTargets()) { - String targetId = target.getId(); - outDirectories.put(targetId, target.getOutputDir()); + incrementalCaches.get(target).clearCacheForRemovedFiles(targetId, removedFiles, target.getOutputDir()); + } - for (String file : KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target)) { - moduleIdsAndSourceFiles.add(new Pair(targetId, new File(file))); - } + IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING; + + for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) { + ModuleBuildTarget target = null; + Collection sourceFiles = outputItem.getSourceFiles(); + if (!sourceFiles.isEmpty()) { + target = sourceToTarget.get(sourceFiles.iterator().next()); } - cache.clearCacheForRemovedFiles(moduleIdsAndSourceFiles, outDirectories); - IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING; - - for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) { - BuildTarget target = null; - Collection sourceFiles = outputItem.getSourceFiles(); - if (!sourceFiles.isEmpty()) { - target = sourceToTarget.get(sourceFiles.iterator().next()); - } - - if (target == null) { - target = representativeTarget; - } - - File outputFile = outputItem.getOutputFile(); - - if (IncrementalCompilation.ENABLED) { - IncrementalCacheImpl.RecompilationDecision newDecision = cache.saveFileToCache(target.getId(), sourceFiles, outputFile); - recompilationDecision = recompilationDecision.merge(newDecision); - } - - outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles)); + if (target == null) { + target = representativeTarget; } + File outputFile = outputItem.getOutputFile(); + if (IncrementalCompilation.ENABLED) { - if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) { - allCompiledFiles.clear(); - return ExitCode.CHUNK_REBUILD_REQUIRED; - } - if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) { - // TODO should mark dependencies as dirty, as well - FSOperations.markDirty(context, chunk, new FileFilter() { - @Override - public boolean accept(@NotNull File file) { - return !allCompiledFiles.contains(file); - } - }); - } - return ExitCode.ADDITIONAL_PASS_REQUIRED; - } - else { - return ExitCode.OK; + IncrementalCacheImpl.RecompilationDecision newDecision = + incrementalCaches.get(target).saveFileToCache(target.getId(), sourceFiles, outputFile); + recompilationDecision = recompilationDecision.merge(newDecision); } + + outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles)); } - finally { - cache.close(); + + if (IncrementalCompilation.ENABLED) { + if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) { + allCompiledFiles.clear(); + return ExitCode.CHUNK_REBUILD_REQUIRED; + } + if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) { + // TODO should mark dependencies as dirty, as well + FSOperations.markDirty(context, chunk, new FileFilter() { + @Override + public boolean accept(@NotNull File file) { + return !allCompiledFiles.contains(file); + } + }); + } + return ExitCode.ADDITIONAL_PASS_REQUIRED; + } + else { + return ExitCode.OK; } } @@ -420,32 +424,7 @@ public class KotlinBuilder extends ModuleLevelBuilder { @Override public Class loadClass(@NotNull String name) throws ClassNotFoundException { - if (name.startsWith("org.jetbrains.jet.jps.incremental.")) { - return loadClassFromBytes(name); - } - else if (name.startsWith("org.jetbrains.jet.lang.resolve.kotlin.incremental.")) { - return compilerClassLoader.loadClass(name); - } - else { - return jpsPluginClassLoader.loadClass(name); - } - } - - private Class loadClassFromBytes(String name) throws ClassNotFoundException { - String classResource = name.replace('.', '/') + ".class"; - InputStream resource = jpsPluginClassLoader.getResourceAsStream(classResource); - if (resource == null) { - return null; - } - byte[] bytes; - try { - bytes = StreamUtil.loadFromStream(resource); - } - catch (IOException e) { - throw new ClassNotFoundException("Couldn't load class " + name, e); - } - - return defineClass(name, bytes, 0, bytes.length); + return jpsPluginClassLoader.loadClass(name); } } } diff --git a/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt b/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt index 23310decd01..a21edd6ac16 100644 --- a/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt @@ -40,10 +40,13 @@ import org.jetbrains.jet.lang.resolve.java.PackageClassUtils import com.intellij.util.containers.MultiMap import com.intellij.openapi.util.io.FileUtil import java.security.MessageDigest +import org.jetbrains.jps.incremental.storage.StorageOwner +import org.jetbrains.jps.builders.storage.StorageProvider +import java.io.IOException val INLINE_ANNOTATION_DESC = "Lkotlin/inline;" -public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { +public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalCache { class object { val PROTO_MAP = "proto.tab" val CONSTANTS_MAP = "constants.tab" @@ -56,10 +59,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { private val inlineFunctionsMap = InlineFunctionsMap() private val packagePartMap = PackagePartMap() + private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap) + public fun saveFileToCache(moduleId: String, sourceFiles: Collection, classFile: File): RecompilationDecision { val fileBytes = classFile.readBytes() val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(fileBytes) - if (classNameAndHeader == null) return RecompilationDecision.DO_NOTHING + if (classNameAndHeader == null) return DO_NOTHING val (className, header) = classNameAndHeader val annotationDataEncoded = header.annotationData @@ -92,14 +97,14 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { return DO_NOTHING } - public fun clearCacheForRemovedFiles(moduleIdsAndSourceFiles: Collection>, outDirectories: Map) { - for ((moduleId, sourceFile) in moduleIdsAndSourceFiles) { - packagePartMap.remove(moduleId, sourceFile) + public fun clearCacheForRemovedFiles(moduleId: String, removedSourceFiles: Collection, outDirectory: File) { + for (removedSourceFile in removedSourceFiles) { + packagePartMap.remove(moduleId, removedSourceFile) } - inlineFunctionsMap.clearOutdated(outDirectories) - constantsMap.clearOutdated(outDirectories) - protoMap.clearOutdated(outDirectories) + inlineFunctionsMap.clearOutdated(outDirectory) + constantsMap.clearOutdated(outDirectory) + protoMap.clearOutdated(outDirectory) } public override fun getRemovedPackageParts(moduleId: String, compiledSourceFilesToFqName: Map): Collection { @@ -110,16 +115,55 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { return protoMap[moduleId, JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))] } - public override fun close() { - protoMap.close() - constantsMap.close() - inlineFunctionsMap.close() - packagePartMap.close() + override fun flush(memoryCachesOnly: Boolean) { + maps.forEach { it.flush(memoryCachesOnly) } } - private abstract class ClassFileBasedMap { - protected abstract val map: PersistentHashMap + public override fun clean() { + maps.forEach { it.clean() } + } + public override fun close() { + maps.forEach { it.close () } + } + + private abstract class BasicMap { + protected var map: PersistentHashMap = createMap() + + protected abstract fun createMap(): PersistentHashMap + + public fun clean() { + try { + map.close() + } + catch (ignored: IOException) { + } + + PersistentHashMap.deleteFilesStartingWith(map.getBaseFile()!!) + try { + map = createMap() + } + catch (ignored: IOException) { + } + } + + public fun flush(memoryCachesOnly: Boolean) { + if (memoryCachesOnly) { + if (map.isDirty()) { + map.dropMemoryCaches() + } + } + else { + map.force() + } + } + + public fun close() { + map.close() + } + } + + private abstract class ClassFileBasedMap: BasicMap() { protected fun getKeyString(moduleId: String, className: JvmClassName): String { return moduleId + ":" + className.getInternalName() } @@ -129,17 +173,15 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { return Pair(key.substring(0, colon), JvmClassName.byInternalName(key.substring(colon + 1))) } - public fun clearOutdated(outDirectories: Map) { + // TODO may be too expensive, because it traverses all files in out directory + public fun clearOutdated(outDirectory: File) { val keysToRemove = HashSet() - map.processKeys { key -> + map.processKeysWithExistingMapping { key -> val (moduleId, className) = parseKeyString(key!!) - val outDir = outDirectories[moduleId] - if (outDir != null) { - val classFile = File(outDir, FileUtil.toSystemDependentName(className.getInternalName()) + ".class") - if (!classFile.exists()) { - keysToRemove.add(key) - } + val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class") + if (!classFile.exists()) { + keysToRemove.add(key) } true @@ -149,14 +191,10 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { map.remove(key) } } - - public fun close() { - map.close() - } } - private inner class ProtoMap: ClassFileBasedMap() { - override val map: PersistentHashMap = PersistentHashMap( + private inner class ProtoMap: ClassFileBasedMap() { + override fun createMap(): PersistentHashMap = PersistentHashMap( File(baseDir, PROTO_MAP), EnumeratorStringDescriptor(), ByteArrayExternalizer @@ -177,8 +215,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { } } - private inner class ConstantsMap: ClassFileBasedMap() { - override val map: PersistentHashMap> = PersistentHashMap( + private inner class ConstantsMap: ClassFileBasedMap>() { + override fun createMap(): PersistentHashMap> = PersistentHashMap( File(baseDir, CONSTANTS_MAP), EnumeratorStringDescriptor(), ConstantsMapExternalizer @@ -274,8 +312,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { } } - private inner class InlineFunctionsMap: ClassFileBasedMap() { - override val map: PersistentHashMap> = PersistentHashMap( + private inner class InlineFunctionsMap: ClassFileBasedMap>() { + override fun createMap(): PersistentHashMap> = PersistentHashMap( File(baseDir, INLINE_FUNCTIONS), EnumeratorStringDescriptor(), InlineFunctionsMapExternalizer @@ -357,9 +395,9 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { } - private inner class PackagePartMap { + private inner class PackagePartMap: BasicMap() { // Format of serialization to string: --> - private val map: PersistentHashMap = PersistentHashMap( + override fun createMap(): PersistentHashMap = PersistentHashMap( File(baseDir, PACKAGE_PARTS), EnumeratorStringDescriptor(), EnumeratorStringDescriptor() @@ -420,10 +458,6 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { return result } - - public fun close() { - map.close() - } } enum class RecompilationDecision { @@ -437,6 +471,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache { } } +public object IncrementalCacheStorageProvider : StorageProvider() { + override fun createStorage(targetDataDir: File?): IncrementalCacheImpl { + return IncrementalCacheImpl(File(targetDataDir, "kotlin")) + } +} + private fun ByteArray.md5(): Long { val d = MessageDigest.getInstance("MD5").digest(this)!! return ((d[0].toLong() and 0xFFL) diff --git a/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt b/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt index 5b94b207b71..65f6ccb6f89 100644 --- a/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt @@ -17,11 +17,15 @@ package org.jetbrains.jet.jps.incremental import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider -import java.io.File +import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache +import kotlin.properties.Delegates -public class IncrementalCacheProviderImpl: IncrementalCacheProvider { - override fun getIncrementalCache(baseDir: File): IncrementalCache { - return IncrementalCacheImpl(baseDir) +public class IncrementalCacheProviderImpl(caches: Map): IncrementalCacheProvider { + private val idToCache = caches.mapKeys { it.key.getId()!! } + + override fun getIncrementalCache(moduleId: String): IncrementalCache { + return idToCache[moduleId]!! } -} \ No newline at end of file +} +