azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made incremental cache per-module.

Browse Source 
Reused IDEA framework which manages per-module storage.

Original commit: 5fade9a5a6
This commit is contained in:
Evgeny Gerashchenko
2014-08-12 20:30:51 +04:00
parent 26ab5cd48a
commit ab44343e1c
4 changed files with 149 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider
-1
View File
@@ -1 +0,0 @@
org.jetbrains.jet.jps.incremental.IncrementalCacheProviderImpl
jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java
+61 -82
View File
@@ -16,6 +16,7 @@
package org.jetbrains.jet.jps.build;
import com.google.common.collect.Maps;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.StreamUtil;
@@ -24,7 +25,6 @@ import com.intellij.util.Function;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.MultiMap;
import gnu.trove.THashSet;
import kotlin.Pair;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.cli.common.KotlinVersion;
import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments;
@@ -40,17 +40,17 @@ import org.jetbrains.jet.compiler.runner.OutputItemsCollectorImpl;
import org.jetbrains.jet.compiler.runner.SimpleOutputItem;
import org.jetbrains.jet.config.IncrementalCompilation;
import org.jetbrains.jet.jps.JpsKotlinCompilerSettings;
import org.jetbrains.jet.jps.incremental.IncrementalCacheImpl;
import org.jetbrains.jet.jps.incremental.*;
import org.jetbrains.jet.preloading.ClassLoaderFactory;
import org.jetbrains.jet.utils.PathUtil;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.DirtyFilesHolder;
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor;
import org.jetbrains.jps.incremental.*;
import org.jetbrains.jps.incremental.java.JavaBuilder;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.module.JpsModule;
@@ -123,13 +123,22 @@ public class KotlinBuilder extends ModuleLevelBuilder {
File outputDir = representativeTarget.getOutputDir();
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
Map<ModuleBuildTarget, IncrementalCacheImpl> incrementalCaches = Maps.newHashMap();
for (ModuleBuildTarget target : chunk.getTargets()) {
incrementalCaches.put(target, dataManager.getStorage(target, IncrementalCacheStorageProvider.INSTANCE$));
}
IncrementalCacheProviderImpl cacheProvider = new IncrementalCacheProviderImpl(incrementalCaches);
CompilerEnvironment environment = CompilerEnvironment.getEnvironmentFor(
PathUtil.getKotlinPathsForJpsPluginOrJpsTests(), outputDir, new ClassLoaderFactory() {
@Override
public ClassLoader create(ClassLoader compilerClassLoader) {
return new MyClassLoader(compilerClassLoader);
}
}
},
cacheProvider
);
if (!environment.success()) {
environment.reportErrorsTo(messageCollector);
@@ -213,7 +222,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
}
// If there's only one target, this map is empty: get() always returns null, and the representativeTarget will be used below
Map<File, BuildTarget<?>> sourceToTarget = new HashMap<File, BuildTarget<?>>();
Map<File, ModuleBuildTarget> sourceToTarget = new HashMap<File, ModuleBuildTarget>();
if (chunk.getTargets().size() > 1) {
for (ModuleBuildTarget target : chunk.getTargets()) {
for (File file : KotlinSourceFileCollector.getAllKotlinSourceFiles(target)) {
@@ -222,67 +231,62 @@ public class KotlinBuilder extends ModuleLevelBuilder {
}
}
IncrementalCacheImpl cache = new IncrementalCacheImpl(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context));
for (ModuleBuildTarget target : chunk.getTargets()) {
String targetId = target.getId();
try {
List<Pair<String, File>> moduleIdsAndSourceFiles = new ArrayList<Pair<String, File>>();
Map<String, File> outDirectories = new HashMap<String, File>();
List<File> removedFiles = ContainerUtil.map(KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target),
new Function<String, File>() {
@Override
public File fun(String s) {
return new File(s);
}
});
for (ModuleBuildTarget target : chunk.getTargets()) {
String targetId = target.getId();
outDirectories.put(targetId, target.getOutputDir());
incrementalCaches.get(target).clearCacheForRemovedFiles(targetId, removedFiles, target.getOutputDir());
}
for (String file : KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target)) {
moduleIdsAndSourceFiles.add(new Pair<String, File>(targetId, new File(file)));
}
IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
ModuleBuildTarget target = null;
Collection<File> sourceFiles = outputItem.getSourceFiles();
if (!sourceFiles.isEmpty()) {
target = sourceToTarget.get(sourceFiles.iterator().next());
}
cache.clearCacheForRemovedFiles(moduleIdsAndSourceFiles, outDirectories);
IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
BuildTarget<?> target = null;
Collection<File> sourceFiles = outputItem.getSourceFiles();
if (!sourceFiles.isEmpty()) {
target = sourceToTarget.get(sourceFiles.iterator().next());
}
if (target == null) {
target = representativeTarget;
}
File outputFile = outputItem.getOutputFile();
if (IncrementalCompilation.ENABLED) {
IncrementalCacheImpl.RecompilationDecision newDecision = cache.saveFileToCache(target.getId(), sourceFiles, outputFile);
recompilationDecision = recompilationDecision.merge(newDecision);
}
outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
if (target == null) {
target = representativeTarget;
}
File outputFile = outputItem.getOutputFile();
if (IncrementalCompilation.ENABLED) {
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
allCompiledFiles.clear();
return ExitCode.CHUNK_REBUILD_REQUIRED;
}
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
// TODO should mark dependencies as dirty, as well
FSOperations.markDirty(context, chunk, new FileFilter() {
@Override
public boolean accept(@NotNull File file) {
return !allCompiledFiles.contains(file);
}
});
}
return ExitCode.ADDITIONAL_PASS_REQUIRED;
}
else {
return ExitCode.OK;
IncrementalCacheImpl.RecompilationDecision newDecision =
incrementalCaches.get(target).saveFileToCache(target.getId(), sourceFiles, outputFile);
recompilationDecision = recompilationDecision.merge(newDecision);
}
outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
}
finally {
cache.close();
if (IncrementalCompilation.ENABLED) {
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
allCompiledFiles.clear();
return ExitCode.CHUNK_REBUILD_REQUIRED;
}
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
// TODO should mark dependencies as dirty, as well
FSOperations.markDirty(context, chunk, new FileFilter() {
@Override
public boolean accept(@NotNull File file) {
return !allCompiledFiles.contains(file);
}
});
}
return ExitCode.ADDITIONAL_PASS_REQUIRED;
}
else {
return ExitCode.OK;
}
}
@@ -420,32 +424,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
@Override
public Class<?> loadClass(@NotNull String name) throws ClassNotFoundException {
if (name.startsWith("org.jetbrains.jet.jps.incremental.")) {
return loadClassFromBytes(name);
}
else if (name.startsWith("org.jetbrains.jet.lang.resolve.kotlin.incremental.")) {
return compilerClassLoader.loadClass(name);
}
else {
return jpsPluginClassLoader.loadClass(name);
}
}
private Class<?> loadClassFromBytes(String name) throws ClassNotFoundException {
String classResource = name.replace('.', '/') + ".class";
InputStream resource = jpsPluginClassLoader.getResourceAsStream(classResource);
if (resource == null) {
return null;
}
byte[] bytes;
try {
bytes = StreamUtil.loadFromStream(resource);
}
catch (IOException e) {
throw new ClassNotFoundException("Couldn't load class " + name, e);
}
return defineClass(name, bytes, 0, bytes.length);
return jpsPluginClassLoader.loadClass(name);
}
}
}
jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt
+79 -39
View File
@@ -40,10 +40,13 @@ import org.jetbrains.jet.lang.resolve.java.PackageClassUtils
import com.intellij.util.containers.MultiMap
import com.intellij.openapi.util.io.FileUtil
import java.security.MessageDigest
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.jps.builders.storage.StorageProvider
import java.io.IOException
val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalCache {
class object {
val PROTO_MAP = "proto.tab"
val CONSTANTS_MAP = "constants.tab"
@@ -56,10 +59,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
private val inlineFunctionsMap = InlineFunctionsMap()
private val packagePartMap = PackagePartMap()
private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap)
public fun saveFileToCache(moduleId: String, sourceFiles: Collection<File>, classFile: File): RecompilationDecision {
val fileBytes = classFile.readBytes()
val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(fileBytes)
if (classNameAndHeader == null) return RecompilationDecision.DO_NOTHING
if (classNameAndHeader == null) return DO_NOTHING
val (className, header) = classNameAndHeader
val annotationDataEncoded = header.annotationData
@@ -92,14 +97,14 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return DO_NOTHING
}
public fun clearCacheForRemovedFiles(moduleIdsAndSourceFiles: Collection<Pair<String, File>>, outDirectories: Map<String, File>) {
for ((moduleId, sourceFile) in moduleIdsAndSourceFiles) {
packagePartMap.remove(moduleId, sourceFile)
public fun clearCacheForRemovedFiles(moduleId: String, removedSourceFiles: Collection<File>, outDirectory: File) {
for (removedSourceFile in removedSourceFiles) {
packagePartMap.remove(moduleId, removedSourceFile)
}
inlineFunctionsMap.clearOutdated(outDirectories)
constantsMap.clearOutdated(outDirectories)
protoMap.clearOutdated(outDirectories)
inlineFunctionsMap.clearOutdated(outDirectory)
constantsMap.clearOutdated(outDirectory)
protoMap.clearOutdated(outDirectory)
}
public override fun getRemovedPackageParts(moduleId: String, compiledSourceFilesToFqName: Map<File, String>): Collection<String> {
@@ -110,16 +115,55 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return protoMap[moduleId, JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))]
}
public override fun close() {
protoMap.close()
constantsMap.close()
inlineFunctionsMap.close()
packagePartMap.close()
override fun flush(memoryCachesOnly: Boolean) {
maps.forEach { it.flush(memoryCachesOnly) }
}
private abstract class ClassFileBasedMap {
protected abstract val map: PersistentHashMap<String, *>
public override fun clean() {
maps.forEach { it.clean() }
}
public override fun close() {
maps.forEach { it.close () }
}
private abstract class BasicMap<V> {
protected var map: PersistentHashMap<String, V> = createMap()
protected abstract fun createMap(): PersistentHashMap<String, V>
public fun clean() {
try {
map.close()
}
catch (ignored: IOException) {
}
PersistentHashMap.deleteFilesStartingWith(map.getBaseFile()!!)
try {
map = createMap()
}
catch (ignored: IOException) {
}
}
public fun flush(memoryCachesOnly: Boolean) {
if (memoryCachesOnly) {
if (map.isDirty()) {
map.dropMemoryCaches()
}
}
else {
map.force()
}
}
public fun close() {
map.close()
}
}
private abstract class ClassFileBasedMap<V>: BasicMap<V>() {
protected fun getKeyString(moduleId: String, className: JvmClassName): String {
return moduleId + ":" + className.getInternalName()
}
@@ -129,17 +173,15 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return Pair(key.substring(0, colon), JvmClassName.byInternalName(key.substring(colon + 1)))
}
public fun clearOutdated(outDirectories: Map<String, File>) {
// TODO may be too expensive, because it traverses all files in out directory
public fun clearOutdated(outDirectory: File) {
val keysToRemove = HashSet<String>()
map.processKeys { key ->
map.processKeysWithExistingMapping { key ->
val (moduleId, className) = parseKeyString(key!!)
val outDir = outDirectories[moduleId]
if (outDir != null) {
val classFile = File(outDir, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
if (!classFile.exists()) {
keysToRemove.add(key)
}
val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
if (!classFile.exists()) {
keysToRemove.add(key)
}
true
@@ -149,14 +191,10 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
map.remove(key)
}
}
public fun close() {
map.close()
}
}
private inner class ProtoMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, ByteArray> = PersistentHashMap(
private inner class ProtoMap: ClassFileBasedMap<ByteArray>() {
override fun createMap(): PersistentHashMap<String, ByteArray> = PersistentHashMap(
File(baseDir, PROTO_MAP),
EnumeratorStringDescriptor(),
ByteArrayExternalizer
@@ -177,8 +215,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
private inner class ConstantsMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
private inner class ConstantsMap: ClassFileBasedMap<Map<String, Any>>() {
override fun createMap(): PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
File(baseDir, CONSTANTS_MAP),
EnumeratorStringDescriptor(),
ConstantsMapExternalizer
@@ -274,8 +312,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
private inner class InlineFunctionsMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
private inner class InlineFunctionsMap: ClassFileBasedMap<Map<String, Long>>() {
override fun createMap(): PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
File(baseDir, INLINE_FUNCTIONS),
EnumeratorStringDescriptor(),
InlineFunctionsMapExternalizer
@@ -357,9 +395,9 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
private inner class PackagePartMap {
private inner class PackagePartMap: BasicMap<String>() {
// Format of serialization to string: <module id> <path separator> <source file path> --> <package part JVM internal name>
private val map: PersistentHashMap<String, String> = PersistentHashMap(
override fun createMap(): PersistentHashMap<String, String> = PersistentHashMap(
File(baseDir, PACKAGE_PARTS),
EnumeratorStringDescriptor(),
EnumeratorStringDescriptor()
@@ -420,10 +458,6 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return result
}
public fun close() {
map.close()
}
}
enum class RecompilationDecision {
@@ -437,6 +471,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
public object IncrementalCacheStorageProvider : StorageProvider<IncrementalCacheImpl>() {
override fun createStorage(targetDataDir: File?): IncrementalCacheImpl {
return IncrementalCacheImpl(File(targetDataDir, "kotlin"))
}
}
private fun ByteArray.md5(): Long {
val d = MessageDigest.getInstance("MD5").digest(this)!!
return ((d[0].toLong() and 0xFFL)
jps/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt
+9 -5
View File
@@ -17,11 +17,15 @@
package org.jetbrains.jet.jps.incremental
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider
import java.io.File
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache
import kotlin.properties.Delegates
public class IncrementalCacheProviderImpl: IncrementalCacheProvider {
override fun getIncrementalCache(baseDir: File): IncrementalCache {
return IncrementalCacheImpl(baseDir)
public class IncrementalCacheProviderImpl(caches: Map<ModuleBuildTarget, IncrementalCacheImpl>): IncrementalCacheProvider {
private val idToCache = caches.mapKeys { it.key.getId()!! }
override fun getIncrementalCache(moduleId: String): IncrementalCache {
return idToCache[moduleId]!!
}
}
}