Made incremental cache per-module.

Reused IDEA framework which manages per-module storage.

Original commit: 5fade9a5a6

jps/jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider

@@ -1 +0,0 @@
-org.jetbrains.jet.jps.incremental.IncrementalCacheProviderImpl

jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java

@@ -16,6 +16,7 @@
 
 package org.jetbrains.jet.jps.build;
 
+import com.google.common.collect.Maps;
 import com.intellij.openapi.util.Key;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.io.StreamUtil;
@@ -24,7 +25,6 @@ import com.intellij.util.Function;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.containers.MultiMap;
 import gnu.trove.THashSet;
-import kotlin.Pair;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.cli.common.KotlinVersion;
 import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments;
@@ -40,17 +40,17 @@ import org.jetbrains.jet.compiler.runner.OutputItemsCollectorImpl;
 import org.jetbrains.jet.compiler.runner.SimpleOutputItem;
 import org.jetbrains.jet.config.IncrementalCompilation;
 import org.jetbrains.jet.jps.JpsKotlinCompilerSettings;
-import org.jetbrains.jet.jps.incremental.IncrementalCacheImpl;
+import org.jetbrains.jet.jps.incremental.*;
 import org.jetbrains.jet.preloading.ClassLoaderFactory;
 import org.jetbrains.jet.utils.PathUtil;
 import org.jetbrains.jps.ModuleChunk;
-import org.jetbrains.jps.builders.BuildTarget;
 import org.jetbrains.jps.builders.DirtyFilesHolder;
 import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor;
 import org.jetbrains.jps.incremental.*;
 import org.jetbrains.jps.incremental.java.JavaBuilder;
 import org.jetbrains.jps.incremental.messages.BuildMessage;
 import org.jetbrains.jps.incremental.messages.CompilerMessage;
+import org.jetbrains.jps.incremental.storage.BuildDataManager;
 import org.jetbrains.jps.model.JpsProject;
 import org.jetbrains.jps.model.module.JpsModule;
 
@@ -123,13 +123,22 @@ public class KotlinBuilder extends ModuleLevelBuilder {
 
         File outputDir = representativeTarget.getOutputDir();
 
+        BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
+        Map<ModuleBuildTarget, IncrementalCacheImpl> incrementalCaches = Maps.newHashMap();
+        for (ModuleBuildTarget target : chunk.getTargets()) {
+            incrementalCaches.put(target, dataManager.getStorage(target, IncrementalCacheStorageProvider.INSTANCE$));
+        }
+
+        IncrementalCacheProviderImpl cacheProvider = new IncrementalCacheProviderImpl(incrementalCaches);
+
         CompilerEnvironment environment = CompilerEnvironment.getEnvironmentFor(
                 PathUtil.getKotlinPathsForJpsPluginOrJpsTests(), outputDir, new ClassLoaderFactory() {
                     @Override
                     public ClassLoader create(ClassLoader compilerClassLoader) {
                         return new MyClassLoader(compilerClassLoader);
                     }
-                }
+                },
+                cacheProvider
         );
         if (!environment.success()) {
             environment.reportErrorsTo(messageCollector);
@@ -213,7 +222,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
         }
 
         // If there's only one target, this map is empty: get() always returns null, and the representativeTarget will be used below
-        Map<File, BuildTarget<?>> sourceToTarget = new HashMap<File, BuildTarget<?>>();
+        Map<File, ModuleBuildTarget> sourceToTarget = new HashMap<File, ModuleBuildTarget>();
         if (chunk.getTargets().size() > 1) {
             for (ModuleBuildTarget target : chunk.getTargets()) {
                 for (File file : KotlinSourceFileCollector.getAllKotlinSourceFiles(target)) {
@@ -222,67 +231,62 @@ public class KotlinBuilder extends ModuleLevelBuilder {
             }
         }
 
-        IncrementalCacheImpl cache = new IncrementalCacheImpl(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context));
+        for (ModuleBuildTarget target : chunk.getTargets()) {
+            String targetId = target.getId();
 
-        try {
+            List<File> removedFiles = ContainerUtil.map(KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target),
-            List<Pair<String, File>> moduleIdsAndSourceFiles = new ArrayList<Pair<String, File>>();
+                                                        new Function<String, File>() {
-            Map<String, File> outDirectories = new HashMap<String, File>();
+                                                            @Override
+                                                            public File fun(String s) {
+                                                                return new File(s);
+                                                            }
+                                                        });
 
-            for (ModuleBuildTarget target : chunk.getTargets()) {
+            incrementalCaches.get(target).clearCacheForRemovedFiles(targetId, removedFiles, target.getOutputDir());
-                String targetId = target.getId();
+        }
-                outDirectories.put(targetId, target.getOutputDir());
 
-                for (String file : KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target)) {
+        IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
-                    moduleIdsAndSourceFiles.add(new Pair<String, File>(targetId, new File(file)));
+
-                }
+        for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
+            ModuleBuildTarget target = null;
+            Collection<File> sourceFiles = outputItem.getSourceFiles();
+            if (!sourceFiles.isEmpty()) {
+                target = sourceToTarget.get(sourceFiles.iterator().next());
             }
-            cache.clearCacheForRemovedFiles(moduleIdsAndSourceFiles, outDirectories);
 
-            IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
+            if (target == null) {
-
+                target = representativeTarget;
-            for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
-                BuildTarget<?> target = null;
-                Collection<File> sourceFiles = outputItem.getSourceFiles();
-                if (!sourceFiles.isEmpty()) {
-                    target = sourceToTarget.get(sourceFiles.iterator().next());
-                }
-
-                if (target == null) {
-                    target = representativeTarget;
-                }
-
-                File outputFile = outputItem.getOutputFile();
-
-                if (IncrementalCompilation.ENABLED) {
-                    IncrementalCacheImpl.RecompilationDecision newDecision = cache.saveFileToCache(target.getId(), sourceFiles, outputFile);
-                    recompilationDecision = recompilationDecision.merge(newDecision);
-                }
-
-                outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
             }
 
+            File outputFile = outputItem.getOutputFile();
+
             if (IncrementalCompilation.ENABLED) {
-                if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
+                IncrementalCacheImpl.RecompilationDecision newDecision =
-                    allCompiledFiles.clear();
+                        incrementalCaches.get(target).saveFileToCache(target.getId(), sourceFiles, outputFile);
-                    return ExitCode.CHUNK_REBUILD_REQUIRED;
+                recompilationDecision = recompilationDecision.merge(newDecision);
-                }
-                if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
-                    // TODO should mark dependencies as dirty, as well
-                    FSOperations.markDirty(context, chunk, new FileFilter() {
-                        @Override
-                        public boolean accept(@NotNull File file) {
-                            return !allCompiledFiles.contains(file);
-                        }
-                    });
-                }
-                return ExitCode.ADDITIONAL_PASS_REQUIRED;
-            }
-            else {
-                return ExitCode.OK;
             }
+
+            outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
         }
-        finally {
+
-            cache.close();
+        if (IncrementalCompilation.ENABLED) {
+            if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
+                allCompiledFiles.clear();
+                return ExitCode.CHUNK_REBUILD_REQUIRED;
+            }
+            if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
+                // TODO should mark dependencies as dirty, as well
+                FSOperations.markDirty(context, chunk, new FileFilter() {
+                    @Override
+                    public boolean accept(@NotNull File file) {
+                        return !allCompiledFiles.contains(file);
+                    }
+                });
+            }
+            return ExitCode.ADDITIONAL_PASS_REQUIRED;
+        }
+        else {
+            return ExitCode.OK;
         }
     }
 
@@ -420,32 +424,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
 
         @Override
         public Class<?> loadClass(@NotNull String name) throws ClassNotFoundException {
-            if (name.startsWith("org.jetbrains.jet.jps.incremental.")) {
+            return jpsPluginClassLoader.loadClass(name);
-                return loadClassFromBytes(name);
-            }
-            else if (name.startsWith("org.jetbrains.jet.lang.resolve.kotlin.incremental.")) {
-                return compilerClassLoader.loadClass(name);
-            }
-            else {
-                return jpsPluginClassLoader.loadClass(name);
-            }
-        }
-
-        private Class<?> loadClassFromBytes(String name) throws ClassNotFoundException {
-            String classResource = name.replace('.', '/') + ".class";
-            InputStream resource = jpsPluginClassLoader.getResourceAsStream(classResource);
-            if (resource == null) {
-                return null;
-            }
-            byte[] bytes;
-            try {
-                bytes = StreamUtil.loadFromStream(resource);
-            }
-            catch (IOException e) {
-                throw new ClassNotFoundException("Couldn't load class " + name, e);
-            }
-
-            return defineClass(name, bytes, 0, bytes.length);
         }
     }
 }

jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt

@@ -40,10 +40,13 @@ import org.jetbrains.jet.lang.resolve.java.PackageClassUtils
 import com.intellij.util.containers.MultiMap
 import com.intellij.openapi.util.io.FileUtil
 import java.security.MessageDigest
+import org.jetbrains.jps.incremental.storage.StorageOwner
+import org.jetbrains.jps.builders.storage.StorageProvider
+import java.io.IOException
 
 val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
 
-public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
+public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalCache {
     class object {
         val PROTO_MAP = "proto.tab"
         val CONSTANTS_MAP = "constants.tab"
@@ -56,10 +59,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
     private val inlineFunctionsMap = InlineFunctionsMap()
     private val packagePartMap = PackagePartMap()
 
+    private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap)
+
     public fun saveFileToCache(moduleId: String, sourceFiles: Collection<File>, classFile: File): RecompilationDecision {
         val fileBytes = classFile.readBytes()
         val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(fileBytes)
-        if (classNameAndHeader == null) return RecompilationDecision.DO_NOTHING
+        if (classNameAndHeader == null) return DO_NOTHING
 
         val (className, header) = classNameAndHeader
         val annotationDataEncoded = header.annotationData
@@ -92,14 +97,14 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
         return DO_NOTHING
     }
 
-    public fun clearCacheForRemovedFiles(moduleIdsAndSourceFiles: Collection<Pair<String, File>>, outDirectories: Map<String, File>) {
+    public fun clearCacheForRemovedFiles(moduleId: String, removedSourceFiles: Collection<File>, outDirectory: File) {
-        for ((moduleId, sourceFile) in moduleIdsAndSourceFiles) {
+        for (removedSourceFile in removedSourceFiles) {
-            packagePartMap.remove(moduleId, sourceFile)
+            packagePartMap.remove(moduleId, removedSourceFile)
         }
 
-        inlineFunctionsMap.clearOutdated(outDirectories)
+        inlineFunctionsMap.clearOutdated(outDirectory)
-        constantsMap.clearOutdated(outDirectories)
+        constantsMap.clearOutdated(outDirectory)
-        protoMap.clearOutdated(outDirectories)
+        protoMap.clearOutdated(outDirectory)
     }
 
     public override fun getRemovedPackageParts(moduleId: String, compiledSourceFilesToFqName: Map<File, String>): Collection<String> {
@@ -110,16 +115,55 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
         return protoMap[moduleId, JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))]
     }
 
-    public override fun close() {
+    override fun flush(memoryCachesOnly: Boolean) {
-        protoMap.close()
+        maps.forEach { it.flush(memoryCachesOnly) }
-        constantsMap.close()
-        inlineFunctionsMap.close()
-        packagePartMap.close()
     }
 
-    private abstract class ClassFileBasedMap {
+    public override fun clean() {
-        protected abstract val map: PersistentHashMap<String, *>
+        maps.forEach { it.clean() }
+    }
 
+    public override fun close() {
+        maps.forEach { it.close () }
+    }
+
+    private abstract class BasicMap<V> {
+        protected var map: PersistentHashMap<String, V> = createMap()
+
+        protected abstract fun createMap(): PersistentHashMap<String, V>
+
+        public fun clean() {
+            try {
+                map.close()
+            }
+            catch (ignored: IOException) {
+            }
+
+            PersistentHashMap.deleteFilesStartingWith(map.getBaseFile()!!)
+            try {
+                map = createMap()
+            }
+            catch (ignored: IOException) {
+            }
+        }
+
+        public fun flush(memoryCachesOnly: Boolean) {
+            if (memoryCachesOnly) {
+                if (map.isDirty()) {
+                    map.dropMemoryCaches()
+                }
+            }
+            else {
+                map.force()
+            }
+        }
+
+        public fun close() {
+            map.close()
+        }
+    }
+
+    private abstract class ClassFileBasedMap<V>: BasicMap<V>() {
         protected fun getKeyString(moduleId: String, className: JvmClassName): String {
             return moduleId + ":" + className.getInternalName()
         }
@@ -129,17 +173,15 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
             return Pair(key.substring(0, colon), JvmClassName.byInternalName(key.substring(colon + 1)))
         }
 
-        public fun clearOutdated(outDirectories: Map<String, File>) {
+        // TODO may be too expensive, because it traverses all files in out directory
+        public fun clearOutdated(outDirectory: File) {
             val keysToRemove = HashSet<String>()
 
-            map.processKeys { key ->
+            map.processKeysWithExistingMapping { key ->
                 val (moduleId, className) = parseKeyString(key!!)
-                val outDir = outDirectories[moduleId]
+                val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
-                if (outDir != null) {
+                if (!classFile.exists()) {
-                    val classFile = File(outDir, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
+                    keysToRemove.add(key)
-                    if (!classFile.exists()) {
-                        keysToRemove.add(key)
-                    }
                 }
 
                 true
@@ -149,14 +191,10 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
                 map.remove(key)
             }
         }
-
-        public fun close() {
-            map.close()
-        }
     }
 
-    private inner class ProtoMap: ClassFileBasedMap() {
+    private inner class ProtoMap: ClassFileBasedMap<ByteArray>() {
-        override val map: PersistentHashMap<String, ByteArray> = PersistentHashMap(
+        override fun createMap(): PersistentHashMap<String, ByteArray> = PersistentHashMap(
                 File(baseDir, PROTO_MAP),
                 EnumeratorStringDescriptor(),
                 ByteArrayExternalizer
@@ -177,8 +215,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
         }
     }
 
-    private inner class ConstantsMap: ClassFileBasedMap() {
+    private inner class ConstantsMap: ClassFileBasedMap<Map<String, Any>>() {
-        override val map: PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
+        override fun createMap(): PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
                 File(baseDir, CONSTANTS_MAP),
                 EnumeratorStringDescriptor(),
                 ConstantsMapExternalizer
@@ -274,8 +312,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
         }
     }
 
-    private inner class InlineFunctionsMap: ClassFileBasedMap() {
+    private inner class InlineFunctionsMap: ClassFileBasedMap<Map<String, Long>>() {
-        override val map: PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
+        override fun createMap(): PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
                 File(baseDir, INLINE_FUNCTIONS),
                 EnumeratorStringDescriptor(),
                 InlineFunctionsMapExternalizer
@@ -357,9 +395,9 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
     }
 
 
-    private inner class PackagePartMap {
+    private inner class PackagePartMap: BasicMap<String>() {
         // Format of serialization to string: <module id> <path separator> <source file path>  -->  <package part JVM internal name>
-        private val map: PersistentHashMap<String, String> = PersistentHashMap(
+        override fun createMap(): PersistentHashMap<String, String> = PersistentHashMap(
                 File(baseDir, PACKAGE_PARTS),
                 EnumeratorStringDescriptor(),
                 EnumeratorStringDescriptor()
@@ -420,10 +458,6 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
 
             return result
         }
-
-        public fun close() {
-            map.close()
-        }
     }
 
     enum class RecompilationDecision {
@@ -437,6 +471,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
     }
 }
 
+public object IncrementalCacheStorageProvider : StorageProvider<IncrementalCacheImpl>() {
+    override fun createStorage(targetDataDir: File?): IncrementalCacheImpl {
+        return IncrementalCacheImpl(File(targetDataDir, "kotlin"))
+    }
+}
+
 private fun ByteArray.md5(): Long {
     val d = MessageDigest.getInstance("MD5").digest(this)!!
     return ((d[0].toLong() and 0xFFL)

jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt

@@ -17,11 +17,15 @@
 package org.jetbrains.jet.jps.incremental
 
 import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider
-import java.io.File
+import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache
+import kotlin.properties.Delegates
 
-public class IncrementalCacheProviderImpl: IncrementalCacheProvider {
+public class IncrementalCacheProviderImpl(caches: Map<ModuleBuildTarget, IncrementalCacheImpl>): IncrementalCacheProvider {
-    override fun getIncrementalCache(baseDir: File): IncrementalCache {
+    private val idToCache = caches.mapKeys { it.key.getId()!! }
-        return IncrementalCacheImpl(baseDir)
+
+    override fun getIncrementalCache(moduleId: String): IncrementalCache {
+        return idToCache[moduleId]!!
     }
-}
+}
+