azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made incremental cache per-module.

Browse Source 
Reused IDEA framework which manages per-module storage.
This commit is contained in:
Evgeny Gerashchenko
2014-08-12 20:30:51 +04:00
parent ff2e2d0203
commit 5fade9a5a6
16 changed files with 241 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-tools/ant/src/org/jetbrains/jet/buildtools/ant/Kotlin2JsCompilerTask.kt
+1 -1
View File
@@ -89,7 +89,7 @@ public class Kotlin2JsCompilerTask : Task() {
log("Compiling ${arguments.freeArgs} => [${arguments.outputFile}]");
val compiler = K2JSCompiler()
val exitCode = compiler.exec(MessageCollectorPlainTextToStream.PLAIN_TEXT_TO_SYSTEM_ERR, arguments)
val exitCode = compiler.exec(MessageCollectorPlainTextToStream.PLAIN_TEXT_TO_SYSTEM_ERR, mapOf(), arguments)
if (exitCode != ExitCode.OK) {
throw BuildException("Compilation finished with exit code $exitCode")
compiler/cli/src/org/jetbrains/jet/cli/common/CLICompiler.java
+20 -8
View File
@@ -29,7 +29,9 @@ import org.jetbrains.jet.cli.jvm.compiler.CompileEnvironmentException;
import org.jetbrains.jet.config.CompilerConfiguration;
import java.io.PrintStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.jetbrains.jet.cli.common.ExitCode.*;
@@ -48,13 +50,13 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
@NotNull
public ExitCode exec(@NotNull PrintStream errStream, @NotNull String... args) {
return exec(errStream, MessageRenderer.PLAIN_WITH_RELATIVE_PATH, args);
return exec(errStream, Collections.<Class, Object>emptyMap(), MessageRenderer.PLAIN_WITH_RELATIVE_PATH, args);
}
@SuppressWarnings("UnusedDeclaration") // Used via reflection in CompilerRunnerUtil#invokeExecMethod
@NotNull
public ExitCode execAndOutputHtml(@NotNull PrintStream errStream, @NotNull String... args) {
return exec(errStream, MessageRenderer.TAGS, args);
public ExitCode execAndOutputHtml(@NotNull PrintStream errStream, @NotNull Map<Class, Object> services, @NotNull String... args) {
return exec(errStream, services, MessageRenderer.TAGS, args);
}
@Nullable
@@ -97,7 +99,12 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
protected abstract A createArguments();
@NotNull
private ExitCode exec(@NotNull PrintStream errStream, @NotNull MessageRenderer messageRenderer, @NotNull String[] args) {
private ExitCode exec(
@NotNull PrintStream errStream,
@NotNull Map<Class, Object> services,
@NotNull MessageRenderer messageRenderer,
@NotNull String[] args
) {
A arguments = parseArguments(errStream, messageRenderer, args);
if (arguments == null) {
return INTERNAL_ERROR;
@@ -119,7 +126,7 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
}
try {
return exec(collector, arguments);
return exec(collector, services, arguments);
}
finally {
errStream.print(messageRenderer.renderConclusion());
@@ -127,13 +134,13 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
}
@NotNull
public ExitCode exec(@NotNull MessageCollector messageCollector, @NotNull A arguments) {
public ExitCode exec(@NotNull MessageCollector messageCollector, @NotNull Map<Class, Object> services, @NotNull A arguments) {
GroupingMessageCollector groupingCollector = new GroupingMessageCollector(messageCollector);
try {
Disposable rootDisposable = Disposer.newDisposable();
try {
MessageSeverityCollector severityCollector = new MessageSeverityCollector(groupingCollector);
ExitCode code = doExecute(arguments, severityCollector, rootDisposable);
ExitCode code = doExecute(arguments, services, severityCollector, rootDisposable);
return severityCollector.anyReported(CompilerMessageSeverity.ERROR) ? COMPILATION_ERROR : code;
}
finally {
@@ -151,7 +158,12 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
}
@NotNull
protected abstract ExitCode doExecute(@NotNull A arguments, @NotNull MessageCollector messageCollector, @NotNull Disposable rootDisposable);
protected abstract ExitCode doExecute(
@NotNull A arguments,
@NotNull Map<Class, Object> services,
@NotNull MessageCollector messageCollector,
@NotNull Disposable rootDisposable
);
protected void printVersionIfNeeded(
@NotNull PrintStream errStream,
compiler/cli/src/org/jetbrains/jet/cli/js/K2JSCompiler.java
+2
View File
@@ -51,6 +51,7 @@ import org.jetbrains.k2js.facade.MainCallParameters;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.jetbrains.jet.cli.common.ExitCode.COMPILATION_ERROR;
import static org.jetbrains.jet.cli.common.ExitCode.OK;
@@ -73,6 +74,7 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
@Override
protected ExitCode doExecute(
@NotNull K2JSCompilerArguments arguments,
@NotNull Map<Class, Object> services,
@NotNull MessageCollector messageCollector,
@NotNull Disposable rootDisposable
) {
compiler/cli/src/org/jetbrains/jet/cli/jvm/JVMConfigurationKeys.java
+4
View File
@@ -18,6 +18,7 @@ package org.jetbrains.jet.cli.jvm;
import org.jetbrains.jet.config.CompilerConfigurationKey;
import org.jetbrains.jet.lang.resolve.AnalyzerScriptParameter;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.cache.IncrementalCacheProvider;
import java.io.File;
import java.util.List;
@@ -43,6 +44,9 @@ public class JVMConfigurationKeys {
public static final CompilerConfigurationKey<File> INCREMENTAL_CACHE_BASE_DIR =
CompilerConfigurationKey.create("incremental cache base dir");
public static final CompilerConfigurationKey<IncrementalCacheProvider> INCREMENTAL_CACHE_PROVIDER =
CompilerConfigurationKey.create("incremental cache provider");
public static final CompilerConfigurationKey<List<String>> MODULE_IDS =
CompilerConfigurationKey.create("module id strings");
}
compiler/cli/src/org/jetbrains/jet/cli/jvm/K2JVMCompiler.java
+8
View File
@@ -35,6 +35,7 @@ import org.jetbrains.jet.codegen.CompilationException;
import org.jetbrains.jet.config.CommonConfigurationKeys;
import org.jetbrains.jet.config.CompilerConfiguration;
import org.jetbrains.jet.lang.resolve.AnalyzerScriptParameter;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.cache.IncrementalCacheProvider;
import org.jetbrains.jet.utils.KotlinPaths;
import org.jetbrains.jet.utils.KotlinPathsFromHomeDir;
import org.jetbrains.jet.utils.PathUtil;
@@ -42,6 +43,7 @@ import org.jetbrains.jet.utils.PathUtil;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static com.google.common.base.Predicates.in;
import static org.jetbrains.jet.cli.common.ExitCode.*;
@@ -57,6 +59,7 @@ public class K2JVMCompiler extends CLICompiler<K2JVMCompilerArguments> {
@NotNull
protected ExitCode doExecute(
@NotNull K2JVMCompilerArguments arguments,
@NotNull Map<Class, Object> services,
@NotNull MessageCollector messageCollector,
@NotNull Disposable rootDisposable
) {
@@ -69,6 +72,11 @@ public class K2JVMCompiler extends CLICompiler<K2JVMCompilerArguments> {
CompilerConfiguration configuration = new CompilerConfiguration();
IncrementalCacheProvider incrementalCacheProvider = (IncrementalCacheProvider) services.get(IncrementalCacheProvider.class);
if (incrementalCacheProvider != null) {
configuration.put(JVMConfigurationKeys.INCREMENTAL_CACHE_PROVIDER, incrementalCacheProvider);
}
try {
configuration.addAll(JVMConfigurationKeys.CLASSPATH_KEY, getClasspath(paths, arguments));
configuration.addAll(JVMConfigurationKeys.ANNOTATIONS_PATH_KEY, getAnnotationsPath(paths, arguments));
compiler/cli/src/org/jetbrains/jet/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.java
+6 -29
View File
@@ -57,7 +57,6 @@ import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalPackage;
import org.jetbrains.jet.lang.resolve.name.FqName;
import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns;
import org.jetbrains.jet.plugin.MainFunctionDetector;
import org.jetbrains.jet.utils.KotlinPaths;
@@ -291,22 +290,6 @@ public class KotlinToJVMBytecodeCompiler {
CliLightClassGenerationSupport support = CliLightClassGenerationSupport.getInstanceForCli(environment.getProject());
BindingTrace sharedTrace = support.getTrace();
ModuleDescriptorImpl sharedModule = support.newModule();
IncrementalCacheProvider incrementalCacheProvider = IncrementalCacheProvider.OBJECT$.getInstance();
File incrementalCacheBaseDir = environment.getConfiguration().get(JVMConfigurationKeys.INCREMENTAL_CACHE_BASE_DIR);
final IncrementalCache incrementalCache;
if (incrementalCacheProvider != null && incrementalCacheBaseDir != null) {
incrementalCache = incrementalCacheProvider.getIncrementalCache(incrementalCacheBaseDir);
Disposer.register(environment.getApplication(), new Disposable() {
@Override
public void dispose() {
incrementalCache.close();
}
});
}
else {
incrementalCache = null;
}
return AnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(
environment.getProject(),
@@ -315,7 +298,7 @@ public class KotlinToJVMBytecodeCompiler {
Predicates.<PsiFile>alwaysTrue(),
sharedModule,
environment.getConfiguration().get(JVMConfigurationKeys.MODULE_IDS),
incrementalCache
environment.getConfiguration().get(JVMConfigurationKeys.INCREMENTAL_CACHE_PROVIDER)
);
}
}
@@ -342,22 +325,16 @@ public class KotlinToJVMBytecodeCompiler {
File outputDirectory
) {
CompilerConfiguration configuration = environment.getConfiguration();
File incrementalCacheDir = configuration.get(JVMConfigurationKeys.INCREMENTAL_CACHE_BASE_DIR);
IncrementalCacheProvider incrementalCacheProvider = IncrementalCacheProvider.OBJECT$.getInstance();
IncrementalCacheProvider incrementalCacheProvider = configuration.get(JVMConfigurationKeys.INCREMENTAL_CACHE_PROVIDER);
Collection<FqName> packagesWithRemovedFiles;
if (incrementalCacheDir == null || moduleId == null || incrementalCacheProvider == null) {
if (moduleId == null || incrementalCacheProvider == null) {
packagesWithRemovedFiles = null;
}
else {
IncrementalCache incrementalCache = incrementalCacheProvider.getIncrementalCache(incrementalCacheDir);
try {
packagesWithRemovedFiles = IncrementalPackage.getPackagesWithRemovedFiles(
incrementalCache, moduleId, environment.getSourceFiles());
}
finally {
incrementalCache.close();
}
IncrementalCache incrementalCache = incrementalCacheProvider.getIncrementalCache(moduleId);
packagesWithRemovedFiles = IncrementalPackage.getPackagesWithRemovedFiles(
incrementalCache, moduleId, environment.getSourceFiles());
}
BindingTraceContext diagnosticHolder = new BindingTraceContext();
GenerationState generationState = new GenerationState(
compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/java/AnalyzerFacadeForJVM.java
+6 -2
View File
@@ -25,6 +25,7 @@ import com.intellij.psi.search.GlobalSearchScope;
import kotlin.Function1;
import kotlin.KotlinPackage;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.analyzer.AnalyzeExhaust;
import org.jetbrains.jet.analyzer.AnalyzerFacade;
import org.jetbrains.jet.context.ContextPackage;
@@ -43,6 +44,7 @@ import org.jetbrains.jet.lang.resolve.TopDownAnalysisParameters;
import org.jetbrains.jet.lang.resolve.java.mapping.JavaToKotlinClassMap;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalPackageFragmentProvider;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider;
import org.jetbrains.jet.lang.resolve.lazy.ResolveSession;
import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactory;
import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactoryService;
@@ -149,7 +151,7 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
Predicate<PsiFile> filesToAnalyzeCompletely,
ModuleDescriptorImpl module,
List<String> moduleIds,
IncrementalCache incrementalCache
@Nullable IncrementalCacheProvider incrementalCacheProvider
) {
GlobalContext globalContext = ContextPackage.GlobalContext();
TopDownAnalysisParameters topDownAnalysisParameters = TopDownAnalysisParameters.create(
@@ -164,8 +166,10 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
try {
List<PackageFragmentProvider> additionalProviders = new ArrayList<PackageFragmentProvider>();
if (incrementalCache != null && moduleIds != null) {
if (moduleIds != null && incrementalCacheProvider != null) {
for (String moduleId : moduleIds) {
IncrementalCache incrementalCache = incrementalCacheProvider.getIncrementalCache(moduleId);
additionalProviders.add(
new IncrementalPackageFragmentProvider(
files, module, globalContext.getStorageManager(), injector.getDeserializationGlobalContextForJava(),
compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/IncrementalCacheProvider.kt
+1 -16
View File
@@ -16,21 +16,6 @@
package org.jetbrains.jet.lang.resolve.kotlin.incremental
import java.io.File
import java.util.ServiceLoader
public trait IncrementalCacheProvider {
// IncrementalCache should be always closed after using
public fun getIncrementalCache(baseDir: File): IncrementalCache
public class object {
public fun getInstance(): IncrementalCacheProvider? {
val serviceLoader = ServiceLoader.load(javaClass<IncrementalCacheProvider>(), javaClass<IncrementalCacheProvider>().getClassLoader())
val implementations = serviceLoader.toList()
if (implementations.size > 1) {
throw IllegalStateException("More than one IncrementalCacheProvider: " + implementations)
}
return implementations.firstOrNull()
}
}
public fun getIncrementalCache(moduleId: String): IncrementalCache
}
compiler/preloader/src/org/jetbrains/jet/preloading/ClassPreloadingUtils.java
+15
View File
@@ -77,6 +77,21 @@ public class ClassPreloadingUtils {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
// When compiler is invoked from JPS, we should use loaded incremental cache interface from its class loader,
// because implementation is loaded from it, as well
if (name.startsWith("org.jetbrains.jet.lang.resolve.kotlin.incremental.cache.IncrementalCache")) {
if (parent == null) {
return super.loadClass(name);
}
try {
return parent.loadClass(name);
}
catch (ClassNotFoundException e) {
return super.loadClass(name);
}
}
// Look in this class loader and then in the parent one
Class<?> aClass = super.loadClass(name);
if (aClass == null) {
compiler/tests/org/jetbrains/jet/lang/resolve/lazy/JvmResolveUtil.java
+1 -2
View File
@@ -85,7 +85,6 @@ public class JvmResolveUtil {
lightClassGenerationSupport.setModule(module);
}
return AnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(project, files, bindingTraceContext, filesToAnalyzeCompletely,
module,
null, null);
module, null, null);
}
}
ide-compiler-runner/src/org/jetbrains/jet/compiler/runner/CompilerEnvironment.java
+25 -3
View File
@@ -24,6 +24,8 @@ import org.jetbrains.jet.utils.KotlinPaths;
import org.jetbrains.jet.utils.PathUtil;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import static org.jetbrains.jet.cli.common.messages.CompilerMessageLocation.NO_LOCATION;
import static org.jetbrains.jet.cli.common.messages.CompilerMessageSeverity.ERROR;
@@ -33,9 +35,16 @@ public final class CompilerEnvironment {
public static CompilerEnvironment getEnvironmentFor(
@NotNull KotlinPaths kotlinPaths,
@Nullable File outputDir,
@Nullable ClassLoaderFactory parentFactory
@Nullable ClassLoaderFactory parentFactory,
@NotNull Object... serviceImplementations
) {
return new CompilerEnvironment(kotlinPaths, outputDir, parentFactory);
Map<Class, Object> servicesMap = new HashMap<Class, Object>();
for (Object serviceImplementation : serviceImplementations) {
for (Class<?> serviceInterface : serviceImplementation.getClass().getInterfaces()) {
servicesMap.put(serviceInterface, serviceImplementation);
}
}
return new CompilerEnvironment(kotlinPaths, outputDir, parentFactory, servicesMap);
}
@NotNull
@@ -44,11 +53,19 @@ public final class CompilerEnvironment {
private final File output;
@Nullable
private final ClassLoaderFactory parentFactory;
@NotNull
private final Map<Class, Object> services;
private CompilerEnvironment(@NotNull KotlinPaths kotlinPaths, @Nullable File output, @Nullable ClassLoaderFactory parentFactory) {
private CompilerEnvironment(
@NotNull KotlinPaths kotlinPaths,
@Nullable File output,
@Nullable ClassLoaderFactory parentFactory,
@NotNull Map<Class, Object> services
) {
this.kotlinPaths = kotlinPaths;
this.output = output;
this.parentFactory = parentFactory;
this.services = services;
}
public boolean success() {
@@ -80,4 +97,9 @@ public final class CompilerEnvironment {
"or specify " + PathUtil.JPS_KOTLIN_HOME_PROPERTY + " system property", NO_LOCATION);
}
}
@NotNull
public Map<Class, Object> getServices() {
return services;
}
}
ide-compiler-runner/src/org/jetbrains/jet/compiler/runner/CompilerRunnerUtil.java
+3 -2
View File
@@ -34,6 +34,7 @@ import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.jetbrains.jet.cli.common.messages.CompilerMessageLocation.NO_LOCATION;
import static org.jetbrains.jet.cli.common.messages.CompilerMessageSeverity.ERROR;
@@ -123,9 +124,9 @@ public class CompilerRunnerUtil {
: getOrCreateClassLoader(environment.getKotlinPaths(), messageCollector);
Class<?> kompiler = Class.forName(compilerClassName, true, loader);
Method exec = kompiler.getMethod("execAndOutputHtml", PrintStream.class, String[].class);
Method exec = kompiler.getMethod("execAndOutputHtml", PrintStream.class, Map.class, String[].class);
return exec.invoke(kompiler.newInstance(), out, arguments);
return exec.invoke(kompiler.newInstance(), out, environment.getServices(), arguments);
}
public static void outputCompilerMessagesAndHandleExitCode(@NotNull MessageCollector messageCollector,
jps-plugin/src/META-INF/services/org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider
-1
View File
@@ -1 +0,0 @@
org.jetbrains.jet.jps.incremental.IncrementalCacheProviderImpl
jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java
+61 -82
View File
@@ -16,6 +16,7 @@
package org.jetbrains.jet.jps.build;
import com.google.common.collect.Maps;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.StreamUtil;
@@ -24,7 +25,6 @@ import com.intellij.util.Function;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.MultiMap;
import gnu.trove.THashSet;
import kotlin.Pair;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.cli.common.KotlinVersion;
import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments;
@@ -40,17 +40,17 @@ import org.jetbrains.jet.compiler.runner.OutputItemsCollectorImpl;
import org.jetbrains.jet.compiler.runner.SimpleOutputItem;
import org.jetbrains.jet.config.IncrementalCompilation;
import org.jetbrains.jet.jps.JpsKotlinCompilerSettings;
import org.jetbrains.jet.jps.incremental.IncrementalCacheImpl;
import org.jetbrains.jet.jps.incremental.*;
import org.jetbrains.jet.preloading.ClassLoaderFactory;
import org.jetbrains.jet.utils.PathUtil;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.DirtyFilesHolder;
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor;
import org.jetbrains.jps.incremental.*;
import org.jetbrains.jps.incremental.java.JavaBuilder;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.module.JpsModule;
@@ -123,13 +123,22 @@ public class KotlinBuilder extends ModuleLevelBuilder {
File outputDir = representativeTarget.getOutputDir();
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
Map<ModuleBuildTarget, IncrementalCacheImpl> incrementalCaches = Maps.newHashMap();
for (ModuleBuildTarget target : chunk.getTargets()) {
incrementalCaches.put(target, dataManager.getStorage(target, IncrementalCacheStorageProvider.INSTANCE$));
}
IncrementalCacheProviderImpl cacheProvider = new IncrementalCacheProviderImpl(incrementalCaches);
CompilerEnvironment environment = CompilerEnvironment.getEnvironmentFor(
PathUtil.getKotlinPathsForJpsPluginOrJpsTests(), outputDir, new ClassLoaderFactory() {
@Override
public ClassLoader create(ClassLoader compilerClassLoader) {
return new MyClassLoader(compilerClassLoader);
}
}
},
cacheProvider
);
if (!environment.success()) {
environment.reportErrorsTo(messageCollector);
@@ -213,7 +222,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
}
// If there's only one target, this map is empty: get() always returns null, and the representativeTarget will be used below
Map<File, BuildTarget<?>> sourceToTarget = new HashMap<File, BuildTarget<?>>();
Map<File, ModuleBuildTarget> sourceToTarget = new HashMap<File, ModuleBuildTarget>();
if (chunk.getTargets().size() > 1) {
for (ModuleBuildTarget target : chunk.getTargets()) {
for (File file : KotlinSourceFileCollector.getAllKotlinSourceFiles(target)) {
@@ -222,67 +231,62 @@ public class KotlinBuilder extends ModuleLevelBuilder {
}
}
IncrementalCacheImpl cache = new IncrementalCacheImpl(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context));
for (ModuleBuildTarget target : chunk.getTargets()) {
String targetId = target.getId();
try {
List<Pair<String, File>> moduleIdsAndSourceFiles = new ArrayList<Pair<String, File>>();
Map<String, File> outDirectories = new HashMap<String, File>();
List<File> removedFiles = ContainerUtil.map(KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target),
new Function<String, File>() {
@Override
public File fun(String s) {
return new File(s);
}
});
for (ModuleBuildTarget target : chunk.getTargets()) {
String targetId = target.getId();
outDirectories.put(targetId, target.getOutputDir());
incrementalCaches.get(target).clearCacheForRemovedFiles(targetId, removedFiles, target.getOutputDir());
}
for (String file : KotlinSourceFileCollector.getRemovedKotlinFiles(dirtyFilesHolder, target)) {
moduleIdsAndSourceFiles.add(new Pair<String, File>(targetId, new File(file)));
}
IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
ModuleBuildTarget target = null;
Collection<File> sourceFiles = outputItem.getSourceFiles();
if (!sourceFiles.isEmpty()) {
target = sourceToTarget.get(sourceFiles.iterator().next());
}
cache.clearCacheForRemovedFiles(moduleIdsAndSourceFiles, outDirectories);
IncrementalCacheImpl.RecompilationDecision recompilationDecision = IncrementalCacheImpl.RecompilationDecision.DO_NOTHING;
for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
BuildTarget<?> target = null;
Collection<File> sourceFiles = outputItem.getSourceFiles();
if (!sourceFiles.isEmpty()) {
target = sourceToTarget.get(sourceFiles.iterator().next());
}
if (target == null) {
target = representativeTarget;
}
File outputFile = outputItem.getOutputFile();
if (IncrementalCompilation.ENABLED) {
IncrementalCacheImpl.RecompilationDecision newDecision = cache.saveFileToCache(target.getId(), sourceFiles, outputFile);
recompilationDecision = recompilationDecision.merge(newDecision);
}
outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
if (target == null) {
target = representativeTarget;
}
File outputFile = outputItem.getOutputFile();
if (IncrementalCompilation.ENABLED) {
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
allCompiledFiles.clear();
return ExitCode.CHUNK_REBUILD_REQUIRED;
}
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
// TODO should mark dependencies as dirty, as well
FSOperations.markDirty(context, chunk, new FileFilter() {
@Override
public boolean accept(@NotNull File file) {
return !allCompiledFiles.contains(file);
}
});
}
return ExitCode.ADDITIONAL_PASS_REQUIRED;
}
else {
return ExitCode.OK;
IncrementalCacheImpl.RecompilationDecision newDecision =
incrementalCaches.get(target).saveFileToCache(target.getId(), sourceFiles, outputFile);
recompilationDecision = recompilationDecision.merge(newDecision);
}
outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
}
finally {
cache.close();
if (IncrementalCompilation.ENABLED) {
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.RECOMPILE_ALL) {
allCompiledFiles.clear();
return ExitCode.CHUNK_REBUILD_REQUIRED;
}
if (recompilationDecision == IncrementalCacheImpl.RecompilationDecision.COMPILE_OTHERS) {
// TODO should mark dependencies as dirty, as well
FSOperations.markDirty(context, chunk, new FileFilter() {
@Override
public boolean accept(@NotNull File file) {
return !allCompiledFiles.contains(file);
}
});
}
return ExitCode.ADDITIONAL_PASS_REQUIRED;
}
else {
return ExitCode.OK;
}
}
@@ -420,32 +424,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
@Override
public Class<?> loadClass(@NotNull String name) throws ClassNotFoundException {
if (name.startsWith("org.jetbrains.jet.jps.incremental.")) {
return loadClassFromBytes(name);
}
else if (name.startsWith("org.jetbrains.jet.lang.resolve.kotlin.incremental.")) {
return compilerClassLoader.loadClass(name);
}
else {
return jpsPluginClassLoader.loadClass(name);
}
}
private Class<?> loadClassFromBytes(String name) throws ClassNotFoundException {
String classResource = name.replace('.', '/') + ".class";
InputStream resource = jpsPluginClassLoader.getResourceAsStream(classResource);
if (resource == null) {
return null;
}
byte[] bytes;
try {
bytes = StreamUtil.loadFromStream(resource);
}
catch (IOException e) {
throw new ClassNotFoundException("Couldn't load class " + name, e);
}
return defineClass(name, bytes, 0, bytes.length);
return jpsPluginClassLoader.loadClass(name);
}
}
}
jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt
+79 -39
View File
@@ -40,10 +40,13 @@ import org.jetbrains.jet.lang.resolve.java.PackageClassUtils
import com.intellij.util.containers.MultiMap
import com.intellij.openapi.util.io.FileUtil
import java.security.MessageDigest
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.jps.builders.storage.StorageProvider
import java.io.IOException
val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalCache {
class object {
val PROTO_MAP = "proto.tab"
val CONSTANTS_MAP = "constants.tab"
@@ -56,10 +59,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
private val inlineFunctionsMap = InlineFunctionsMap()
private val packagePartMap = PackagePartMap()
private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap)
public fun saveFileToCache(moduleId: String, sourceFiles: Collection<File>, classFile: File): RecompilationDecision {
val fileBytes = classFile.readBytes()
val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(fileBytes)
if (classNameAndHeader == null) return RecompilationDecision.DO_NOTHING
if (classNameAndHeader == null) return DO_NOTHING
val (className, header) = classNameAndHeader
val annotationDataEncoded = header.annotationData
@@ -92,14 +97,14 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return DO_NOTHING
}
public fun clearCacheForRemovedFiles(moduleIdsAndSourceFiles: Collection<Pair<String, File>>, outDirectories: Map<String, File>) {
for ((moduleId, sourceFile) in moduleIdsAndSourceFiles) {
packagePartMap.remove(moduleId, sourceFile)
public fun clearCacheForRemovedFiles(moduleId: String, removedSourceFiles: Collection<File>, outDirectory: File) {
for (removedSourceFile in removedSourceFiles) {
packagePartMap.remove(moduleId, removedSourceFile)
}
inlineFunctionsMap.clearOutdated(outDirectories)
constantsMap.clearOutdated(outDirectories)
protoMap.clearOutdated(outDirectories)
inlineFunctionsMap.clearOutdated(outDirectory)
constantsMap.clearOutdated(outDirectory)
protoMap.clearOutdated(outDirectory)
}
public override fun getRemovedPackageParts(moduleId: String, compiledSourceFilesToFqName: Map<File, String>): Collection<String> {
@@ -110,16 +115,55 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return protoMap[moduleId, JvmClassName.byFqNameWithoutInnerClasses(PackageClassUtils.getPackageClassFqName(FqName(fqName)))]
}
public override fun close() {
protoMap.close()
constantsMap.close()
inlineFunctionsMap.close()
packagePartMap.close()
override fun flush(memoryCachesOnly: Boolean) {
maps.forEach { it.flush(memoryCachesOnly) }
}
private abstract class ClassFileBasedMap {
protected abstract val map: PersistentHashMap<String, *>
public override fun clean() {
maps.forEach { it.clean() }
}
public override fun close() {
maps.forEach { it.close () }
}
private abstract class BasicMap<V> {
protected var map: PersistentHashMap<String, V> = createMap()
protected abstract fun createMap(): PersistentHashMap<String, V>
public fun clean() {
try {
map.close()
}
catch (ignored: IOException) {
}
PersistentHashMap.deleteFilesStartingWith(map.getBaseFile()!!)
try {
map = createMap()
}
catch (ignored: IOException) {
}
}
public fun flush(memoryCachesOnly: Boolean) {
if (memoryCachesOnly) {
if (map.isDirty()) {
map.dropMemoryCaches()
}
}
else {
map.force()
}
}
public fun close() {
map.close()
}
}
private abstract class ClassFileBasedMap<V>: BasicMap<V>() {
protected fun getKeyString(moduleId: String, className: JvmClassName): String {
return moduleId + ":" + className.getInternalName()
}
@@ -129,17 +173,15 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return Pair(key.substring(0, colon), JvmClassName.byInternalName(key.substring(colon + 1)))
}
public fun clearOutdated(outDirectories: Map<String, File>) {
// TODO may be too expensive, because it traverses all files in out directory
public fun clearOutdated(outDirectory: File) {
val keysToRemove = HashSet<String>()
map.processKeys { key ->
map.processKeysWithExistingMapping { key ->
val (moduleId, className) = parseKeyString(key!!)
val outDir = outDirectories[moduleId]
if (outDir != null) {
val classFile = File(outDir, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
if (!classFile.exists()) {
keysToRemove.add(key)
}
val classFile = File(outDirectory, FileUtil.toSystemDependentName(className.getInternalName()) + ".class")
if (!classFile.exists()) {
keysToRemove.add(key)
}
true
@@ -149,14 +191,10 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
map.remove(key)
}
}
public fun close() {
map.close()
}
}
private inner class ProtoMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, ByteArray> = PersistentHashMap(
private inner class ProtoMap: ClassFileBasedMap<ByteArray>() {
override fun createMap(): PersistentHashMap<String, ByteArray> = PersistentHashMap(
File(baseDir, PROTO_MAP),
EnumeratorStringDescriptor(),
ByteArrayExternalizer
@@ -177,8 +215,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
private inner class ConstantsMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
private inner class ConstantsMap: ClassFileBasedMap<Map<String, Any>>() {
override fun createMap(): PersistentHashMap<String, Map<String, Any>> = PersistentHashMap(
File(baseDir, CONSTANTS_MAP),
EnumeratorStringDescriptor(),
ConstantsMapExternalizer
@@ -274,8 +312,8 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
private inner class InlineFunctionsMap: ClassFileBasedMap() {
override val map: PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
private inner class InlineFunctionsMap: ClassFileBasedMap<Map<String, Long>>() {
override fun createMap(): PersistentHashMap<String, Map<String, Long>> = PersistentHashMap(
File(baseDir, INLINE_FUNCTIONS),
EnumeratorStringDescriptor(),
InlineFunctionsMapExternalizer
@@ -357,9 +395,9 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
private inner class PackagePartMap {
private inner class PackagePartMap: BasicMap<String>() {
// Format of serialization to string: <module id> <path separator> <source file path> --> <package part JVM internal name>
private val map: PersistentHashMap<String, String> = PersistentHashMap(
override fun createMap(): PersistentHashMap<String, String> = PersistentHashMap(
File(baseDir, PACKAGE_PARTS),
EnumeratorStringDescriptor(),
EnumeratorStringDescriptor()
@@ -420,10 +458,6 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
return result
}
public fun close() {
map.close()
}
}
enum class RecompilationDecision {
@@ -437,6 +471,12 @@ public class IncrementalCacheImpl(val baseDir: File): IncrementalCache {
}
}
public object IncrementalCacheStorageProvider : StorageProvider<IncrementalCacheImpl>() {
override fun createStorage(targetDataDir: File?): IncrementalCacheImpl {
return IncrementalCacheImpl(File(targetDataDir, "kotlin"))
}
}
private fun ByteArray.md5(): Long {
val d = MessageDigest.getInstance("MD5").digest(this)!!
return ((d[0].toLong() and 0xFFL)
jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementlaCacheProviderImpl.kt
+9 -5
View File
@@ -17,11 +17,15 @@
package org.jetbrains.jet.jps.incremental
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCacheProvider
import java.io.File
import org.jetbrains.jps.incremental.ModuleBuildTarget
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache
import kotlin.properties.Delegates
public class IncrementalCacheProviderImpl: IncrementalCacheProvider {
override fun getIncrementalCache(baseDir: File): IncrementalCache {
return IncrementalCacheImpl(baseDir)
public class IncrementalCacheProviderImpl(caches: Map<ModuleBuildTarget, IncrementalCacheImpl>): IncrementalCacheProvider {
private val idToCache = caches.mapKeys { it.key.getId()!! }
override fun getIncrementalCache(moduleId: String): IncrementalCache {
return idToCache[moduleId]!!
}
}
}