Supported cases of removing source files from package fragment.

This commit is contained in:
Evgeny Gerashchenko
2014-06-05 17:54:43 +04:00
parent fddc8edc03
commit 820bd911fb
12 changed files with 181 additions and 40 deletions
@@ -16,17 +16,19 @@
package org.jetbrains.jet.codegen;
import com.google.common.collect.Sets;
import com.intellij.util.containers.MultiMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.org.objectweb.asm.Type;
import org.jetbrains.jet.codegen.state.GenerationState;
import org.jetbrains.jet.lang.psi.JetFile;
import org.jetbrains.jet.lang.psi.JetScript;
import org.jetbrains.jet.lang.resolve.ScriptNameUtil;
import org.jetbrains.jet.lang.resolve.name.FqName;
import org.jetbrains.org.objectweb.asm.Type;
import java.util.Collection;
import java.util.Map;
import java.util.HashSet;
import java.util.Set;
import static org.jetbrains.jet.codegen.binding.CodegenBinding.registerClassNameForScript;
@@ -55,8 +57,9 @@ public class KotlinCodegenFacade {
packageFqNameToFiles.putValue(file.getPackageFqName(), file);
}
for (Map.Entry<FqName, Collection<JetFile>> entry : packageFqNameToFiles.entrySet()) {
generatePackage(state, entry.getKey(), entry.getValue(), errorHandler);
Set<FqName> removedPackageFiles = new HashSet<FqName>(state.getPackagesWithRemovedFiles());
for (FqName fqName : Sets.union(removedPackageFiles, packageFqNameToFiles.keySet())) {
generatePackage(state, fqName, packageFqNameToFiles.get(fqName), errorHandler);
}
state.getFactory().done();
@@ -76,10 +76,12 @@ public class PackageCodegen {
public PackageCodegen(@NotNull GenerationState state, @NotNull Collection<JetFile> files, @NotNull final FqName fqName) {
this.state = state;
this.files = files;
this.packageFragment = getOnlyPackageFragment();
this.compiledPackageFragment = getCompiledPackageFragment(packageFragment);
this.packageFragment = getOnlyPackageFragment(fqName);
this.compiledPackageFragment = getCompiledPackageFragment(fqName);
this.previouslyCompiledCallables = filterDeserializedCallables(compiledPackageFragment);
assert packageFragment != null || compiledPackageFragment != null : fqName.asString() + " " + files;
this.v = new ClassBuilderOnDemand(new Function0<ClassBuilder>() {
@Override
public ClassBuilder invoke() {
@@ -117,14 +119,14 @@ public class PackageCodegen {
}
@Nullable
private static PackageFragmentDescriptor getCompiledPackageFragment(@NotNull PackageFragmentDescriptor packageFragment) {
private PackageFragmentDescriptor getCompiledPackageFragment(@NotNull FqName fqName) {
if (!IncrementalCompilation.ENABLED) {
return null;
}
// TODO rewrite it to something more robust when module system is implemented
for (PackageFragmentDescriptor anotherFragment : packageFragment.getContainingDeclaration().getPackageFragmentProvider()
.getPackageFragments(packageFragment.getFqName())) {
for (PackageFragmentDescriptor anotherFragment : state.getModule().getPackageFragmentProvider()
.getPackageFragments(fqName)) {
if (anotherFragment instanceof IncrementalPackageFragmentProvider.IncrementalPackageFragment) {
return anotherFragment;
}
@@ -212,10 +214,10 @@ public class PackageCodegen {
}
}
if (generateCallableMemberTasks.isEmpty()) return;
generateDelegationsToPreviouslyCompiled(generateCallableMemberTasks);
if (generateCallableMemberTasks.isEmpty()) return;
for (CallableMemberDescriptor member : Ordering.from(MemberComparator.INSTANCE).sortedCopy(generateCallableMemberTasks.keySet())) {
generateCallableMemberTasks.get(member).run();
}
@@ -235,9 +237,9 @@ public class PackageCodegen {
}
DescriptorSerializer serializer = new DescriptorSerializer(new JavaSerializerExtension(bindings));
Collection<PackageFragmentDescriptor> packageFragments = compiledPackageFragment == null
? Collections.singleton(packageFragment)
: Arrays.asList(packageFragment, compiledPackageFragment);
Collection<PackageFragmentDescriptor> packageFragments = Lists.newArrayList();
ContainerUtil.addIfNotNull(packageFragments, packageFragment);
ContainerUtil.addIfNotNull(packageFragments, compiledPackageFragment);
ProtoBuf.Package packageProto = serializer.packageProto(packageFragments).build();
if (packageProto.getMemberCount() == 0) return;
@@ -325,20 +327,27 @@ public class PackageCodegen {
};
}
@NotNull
private PackageFragmentDescriptor getOnlyPackageFragment() {
@Nullable
private PackageFragmentDescriptor getOnlyPackageFragment(@NotNull FqName expectedFqName) {
SmartList<PackageFragmentDescriptor> fragments = new SmartList<PackageFragmentDescriptor>();
for (JetFile file : files) {
PackageFragmentDescriptor fragment = state.getBindingContext().get(BindingContext.FILE_TO_PACKAGE_FRAGMENT, file);
assert fragment != null : "package fragment is null for " + file;
assert expectedFqName.equals(fragment.getFqName()) :
"expected package fq name: " + expectedFqName + ", actual: " + fragment.getFqName();
if (!fragments.contains(fragment)) {
fragments.add(fragment);
}
}
if (fragments.size() != 1) {
if (fragments.size() > 1) {
throw new IllegalStateException("More than one package fragment, files: " + files + " | fragments: " + fragments);
}
if (fragments.isEmpty()) {
return null;
}
return fragments.get(0);
}
@@ -31,7 +31,10 @@ import org.jetbrains.jet.lang.reflect.ReflectionTypes;
import org.jetbrains.jet.lang.resolve.BindingContext;
import org.jetbrains.jet.lang.resolve.BindingTrace;
import org.jetbrains.jet.lang.resolve.DelegatingBindingTrace;
import org.jetbrains.jet.lang.resolve.name.FqName;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public class GenerationState {
@@ -98,6 +101,12 @@ public class GenerationState {
private final JvmFunctionImplTypes functionImplTypes;
@NotNull
private final ModuleDescriptor module;
@NotNull
private final Collection<FqName> packagesWithRemovedFiles;
public GenerationState(
@NotNull Project project,
@NotNull ClassBuilderFactory builderFactory,
@@ -106,7 +115,7 @@ public class GenerationState {
@NotNull List<JetFile> files
) {
this(project, builderFactory, Progress.DEAF, module, bindingContext, files, true, false, GenerateClassFilter.GENERATE_ALL,
InlineCodegenUtil.DEFAULT_INLINE_FLAG);
InlineCodegenUtil.DEFAULT_INLINE_FLAG, null);
}
public GenerationState(
@@ -119,11 +128,14 @@ public class GenerationState {
boolean generateNotNullAssertions,
boolean generateNotNullParamAssertions,
GenerateClassFilter generateClassFilter,
boolean inlineEnabled
boolean inlineEnabled,
@Nullable Collection<FqName> packagesWithRemovedFiles
) {
this.project = project;
this.progress = progress;
this.module = module;
this.files = files;
this.packagesWithRemovedFiles = packagesWithRemovedFiles == null ? Collections.<FqName>emptySet() : packagesWithRemovedFiles;
this.classBuilderMode = builderFactory.getClassBuilderMode();
this.inlineEnabled = inlineEnabled;
@@ -239,4 +251,14 @@ public class GenerationState {
public void setEarlierScriptsForReplInterpreter(@Nullable List<ScriptDescriptor> earlierScriptsForReplInterpreter) {
this.earlierScriptsForReplInterpreter = earlierScriptsForReplInterpreter;
}
@NotNull
public ModuleDescriptor getModule() {
return module;
}
@NotNull
public Collection<FqName> getPackagesWithRemovedFiles() {
return packagesWithRemovedFiles;
}
}
@@ -51,6 +51,7 @@ import org.jetbrains.jet.lang.resolve.BindingTrace;
import org.jetbrains.jet.lang.resolve.ScriptNameUtil;
import org.jetbrains.jet.lang.resolve.java.AnalyzerFacadeForJVM;
import org.jetbrains.jet.lang.resolve.java.PackageClassUtils;
import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
import org.jetbrains.jet.lang.resolve.name.FqName;
import org.jetbrains.jet.plugin.MainFunctionDetector;
import org.jetbrains.jet.utils.KotlinPaths;
@@ -58,6 +59,7 @@ import org.jetbrains.jet.utils.KotlinPaths;
import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -129,7 +131,7 @@ public class KotlinToJVMBytecodeCompiler {
}
}
);
GenerationState generationState = generate(environment, exhaust, jetFiles);
GenerationState generationState = generate(environment, exhaust, jetFiles, module.getModuleName());
outputFiles.put(module, generationState.getFactory());
}
}
@@ -276,7 +278,7 @@ public class KotlinToJVMBytecodeCompiler {
exhaust.throwIfError();
return generate(environment, exhaust, environment.getSourceFiles());
return generate(environment, exhaust, environment.getSourceFiles(), null);
}
@Nullable
@@ -320,17 +322,23 @@ public class KotlinToJVMBytecodeCompiler {
private static GenerationState generate(
@NotNull JetCoreEnvironment environment,
@NotNull AnalyzeExhaust exhaust,
@NotNull List<JetFile> sourceFiles
@NotNull List<JetFile> sourceFiles,
@Nullable String moduleId
) {
CompilerConfiguration configuration = environment.getConfiguration();
File incrementalCacheDir = configuration.get(JVMConfigurationKeys.INCREMENTAL_CACHE_BASE_DIR);
Collection<FqName> packagesWithRemovedFiles =
incrementalCacheDir == null || moduleId == null
? null
: new IncrementalCache(incrementalCacheDir).getPackagesWithRemovedFiles(moduleId, environment.getSourceFiles());
GenerationState generationState = new GenerationState(
environment.getProject(), ClassBuilderFactories.BINARIES, Progress.DEAF,
exhaust.getModuleDescriptor(), exhaust.getBindingContext(), sourceFiles,
configuration.get(JVMConfigurationKeys.GENERATE_NOT_NULL_ASSERTIONS, false),
configuration.get(JVMConfigurationKeys.GENERATE_NOT_NULL_PARAMETER_ASSERTIONS, false),
GenerationState.GenerateClassFilter.GENERATE_ALL,
configuration.get(JVMConfigurationKeys.ENABLE_INLINE, InlineCodegenUtil.DEFAULT_INLINE_FLAG)
);
configuration.get(JVMConfigurationKeys.ENABLE_INLINE, InlineCodegenUtil.DEFAULT_INLINE_FLAG),
packagesWithRemovedFiles);
KotlinCodegenFacade.compileCorrectFiles(generationState, CompilationErrorHandler.THROW_EXCEPTION);
return generationState;
}
@@ -25,8 +25,11 @@ import org.jetbrains.jet.descriptors.serialization.JavaProtoBuf
import org.jetbrains.jet.lang.resolve.kotlin.PackagePartClassUtils
import org.jetbrains.jet.lang.resolve.java.JvmClassName
public class CliSourcesMemberFilter(files: Collection<JetFile>): MemberFilter {
val packagePartClassNames = files.map { PackagePartClassUtils.getPackagePartInternalName(it) }.toSet()
public class CliSourcesMemberFilter(files: Collection<JetFile>, removedPackageParts: Collection<JvmClassName>): MemberFilter {
val packagePartClassNames = (
files.map { PackagePartClassUtils.getPackagePartInternalName(it) }
+ removedPackageParts.map { it.getInternalName() }
).toSet()
override fun acceptPackagePartClass(container: PackageFragmentDescriptor, member: ProtoBuf.Callable, nameResolver: NameResolver): Boolean {
if (member.hasExtension(JavaProtoBuf.implClassName)) {
@@ -36,11 +36,15 @@ import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.jet.lang.resolve.java.JvmAnnotationNames
import org.jetbrains.jet.lang.resolve.java.JvmClassName
import java.util.TreeMap
import java.util.HashSet
import org.jetbrains.jet.lang.psi.JetFile
import java.util.HashMap
public class IncrementalCache(baseDir: File) {
class object {
val PROTO_MAP = "proto.tab"
val CONSTANTS_MAP = "constants.tab"
val PACKAGE_SOURCES = "package-sources.tab"
private fun getConstantsHash(bytes: ByteArray): Int {
val result = TreeMap<String, Any>() // keys order should defined to check hash of a map
@@ -59,7 +63,12 @@ public class IncrementalCache(baseDir: File) {
}
private val protoData: PersistentMap<ClassOrPackageId, ByteArray>
private val constantsData = PersistentHashMap(File(baseDir, CONSTANTS_MAP), EnumeratorStringDescriptor(), IntInlineKeyDescriptor())
private val constantsData: PersistentHashMap<String, Int>
= PersistentHashMap(File(baseDir, CONSTANTS_MAP), EnumeratorStringDescriptor(), IntInlineKeyDescriptor())
// Format of serialization to string: <module id> <path separator> <source file path> --> <package part JVM internal name>
private val packageSourcesData: PersistentHashMap<String, String>
= PersistentHashMap(File(baseDir, PACKAGE_SOURCES), EnumeratorStringDescriptor(), EnumeratorStringDescriptor())
;{
protoData = PersistentHashMap(File(baseDir, PROTO_MAP), object : KeyDescriptor<ClassOrPackageId> {
@@ -96,7 +105,7 @@ public class IncrementalCache(baseDir: File) {
})
}
public fun saveFileToCache(moduleId: String, file: File): Boolean {
public fun saveFileToCache(moduleId: String, sourceFiles: Collection<File>, file: File): Boolean {
val fileBytes = file.readBytes()
val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(fileBytes)
if (classNameAndHeader == null) return false
@@ -116,6 +125,8 @@ public class IncrementalCache(baseDir: File) {
}
}
if (header.syntheticClassKind == JvmAnnotationNames.KotlinSyntheticClass.Kind.PACKAGE_PART) {
assert(sourceFiles.size == 1) { "Package part from several source files: $sourceFiles" }
putPackagePartSourceData(moduleId, sourceFiles.first(), className)
return putConstantsData(className, getConstantsHash(fileBytes))
}
@@ -143,6 +154,57 @@ public class IncrementalCache(baseDir: File) {
return true
}
private fun putPackagePartSourceData(moduleId: String, sourceFile: File, className: JvmClassName?) {
val key = moduleId + File.pathSeparator + sourceFile.getAbsolutePath()
if (className != null) {
packageSourcesData.put(key, className.getInternalName())
}
else {
packageSourcesData.remove(key)
}
}
public fun clearPackagePartSourceData(moduleId: String, sourceFile: File) {
putPackagePartSourceData(moduleId, sourceFile, null)
}
public fun getPackagesWithRemovedFiles(moduleId: String, compiledSourceFiles: Collection<JetFile>): Collection<FqName> {
return getRemovedPackageParts(moduleId, compiledSourceFiles).map { it.getFqNameForClassNameWithoutDollars().parent() }
}
public fun getRemovedPackageParts(moduleId: String, compiledSourceFiles: Collection<JetFile>): Collection<JvmClassName> {
val sourceFileToCurrentFqNameMap = HashMap<File, FqName>()
for (sourceFile in compiledSourceFiles) {
sourceFileToCurrentFqNameMap[File(sourceFile.getVirtualFile()!!.getPath())] = sourceFile.getPackageFqName()
}
val result = HashSet<JvmClassName>()
packageSourcesData.processKeysWithExistingMapping { key ->
if (key!!.startsWith(moduleId + File.pathSeparator)) {
val indexOf = key.indexOf(File.pathSeparator)
val sourceFile = File(key.substring(indexOf + 1))
val packagePartClassName = JvmClassName.byInternalName(packageSourcesData[key]!!)
if (!sourceFile.exists()) {
result.add(packagePartClassName)
}
else {
val previousPackageFqName = packagePartClassName.getFqNameForClassNameWithoutDollars().parent()
val currentPackageFqName = sourceFileToCurrentFqNameMap[sourceFile]
if (currentPackageFqName != null && currentPackageFqName != previousPackageFqName) {
result.add(packagePartClassName)
}
}
}
true
}
return result
}
public fun getPackageData(moduleId: String, fqName: FqName): ByteArray? {
return protoData[ClassOrPackageId(moduleId, fqName)]
}
@@ -150,6 +212,7 @@ public class IncrementalCache(baseDir: File) {
public fun close() {
protoData.close()
constantsData.close()
packageSourcesData.close()
}
private data class ClassOrPackageId(val moduleId: String, val fqName: FqName) {
@@ -52,7 +52,7 @@ public class IncrementalPackageFragmentProvider(
) : PackageFragmentProvider {
val memberFilter = CliSourcesMemberFilter(sourceFiles)
val memberFilter = CliSourcesMemberFilter(sourceFiles, incrementalCache.getRemovedPackageParts(moduleId, sourceFiles))
val fqNameToSubFqNames = MultiMap<FqName, FqName>()
val fqNameToPackageFragment = HashMap<FqName, PackageFragmentDescriptor>()
@@ -76,6 +76,10 @@ public class IncrementalPackageFragmentProvider(
createPackageFragment(source.getPackageFqName())
}
}
for (fqName in incrementalCache.getPackagesWithRemovedFiles(moduleId, sourceFiles)) {
createPackageFragment(fqName)
}
}
override fun getSubPackagesOf(fqName: FqName): Collection<FqName> {
@@ -276,7 +276,8 @@ public class KotlinJavaFileStubProvider<T extends WithFileStub> implements Cache
Lists.newArrayList(files),
/*not-null assertions*/false, false,
/*generateClassFilter=*/stubGenerationStrategy.getGenerateClassFilter(),
/*to generate inline flag on methods*/true
/*to generate inline flag on methods*/true,
null
);
state.beforeCompile();
@@ -62,7 +62,8 @@ public class CodegenTestUtil {
configuration.get(JVMConfigurationKeys.GENERATE_NOT_NULL_ASSERTIONS, true),
configuration.get(JVMConfigurationKeys.GENERATE_NOT_NULL_PARAMETER_ASSERTIONS, true),
GenerationState.GenerateClassFilter.GENERATE_ALL,
configuration.get(JVMConfigurationKeys.ENABLE_INLINE, InlineCodegenUtil.DEFAULT_INLINE_FLAG)
configuration.get(JVMConfigurationKeys.ENABLE_INLINE, InlineCodegenUtil.DEFAULT_INLINE_FLAG),
null
);
KotlinCodegenFacade.compileCorrectFiles(state, CompilationErrorHandler.THROW_EXCEPTION);
return state.getFactory();
@@ -114,7 +114,7 @@ public class KotlinBytecodeToolWindow extends JPanel implements Disposable {
exhaust.getModuleDescriptor(), exhaust.getBindingContext(),
Collections.singletonList(jetFile), true, true,
GenerationState.GenerateClassFilter.GENERATE_ALL,
enableInline.isSelected());
enableInline.isSelected(), null);
KotlinCodegenFacade.compileCorrectFiles(state, CompilationErrorHandler.THROW_EXCEPTION);
}
catch (ProcessCanceledException e) {
@@ -65,6 +65,7 @@ import static org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JvmCom
public class KotlinBuilder extends ModuleLevelBuilder {
private static final Key<Set<File>> ALL_COMPILED_FILES_KEY = Key.create("_all_kotlin_compiled_files_");
private static final Key<Set<ModuleBuildTarget>> PROCESSED_TARGETS_WITH_REMOVED_FILES = Key.create("_processed_targets_with_removed_files_");
public static final String KOTLIN_BUILDER_NAME = "Kotlin Builder";
private static final List<String> COMPILABLE_FILE_EXTENSIONS = Collections.singletonList("kt");
@@ -172,7 +173,20 @@ public class KotlinBuilder extends ModuleLevelBuilder {
filesToCompile.removeAll(allCompiledFiles);
allCompiledFiles.addAll(filesToCompile);
File moduleFile = KotlinBuilderModuleScriptGenerator.generateModuleDescription(context, chunk, filesToCompile);
Set<ModuleBuildTarget> processedTargetsWithRemoved = getProcessedTargetsWithRemovedFilesContainer(context);
boolean haveRemovedFiles = false;
for (ModuleBuildTarget target : chunk.getTargets()) {
if (processedTargetsWithRemoved.add(target)) {
if (!dirtyFilesHolder.getRemovedFiles(target).isEmpty()) {
haveRemovedFiles = true;
break;
}
}
}
File moduleFile = KotlinBuilderModuleScriptGenerator
.generateModuleDescription(context, chunk, filesToCompile, haveRemovedFiles);
if (moduleFile == null) {
// No Kotlin sources found
return ExitCode.NOTHING_DONE;
@@ -198,17 +212,20 @@ public class KotlinBuilder extends ModuleLevelBuilder {
IncrementalCache cache = new IncrementalCache(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context));
try {
for (ModuleBuildTarget target : chunk.getTargets()) {
for (String file : dirtyFilesHolder.getRemovedFiles(target)) {
cache.clearPackagePartSourceData(target.getId(), new File(file));
}
}
boolean significantChanges = false;
for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
BuildTarget<?> target = null;
Collection<File> sourceFiles = outputItem.getSourceFiles();
if (sourceFiles != null && !sourceFiles.isEmpty()) {
if (!sourceFiles.isEmpty()) {
target = sourceToTarget.get(sourceFiles.iterator().next());
}
else {
messageCollector.report(EXCEPTION, "KotlinBuilder: outputItem.sourceFiles is null or empty, outputItem = " + outputItem, NO_LOCATION);
}
if (target == null) {
target = representativeTarget;
@@ -217,7 +234,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
File outputFile = outputItem.getOutputFile();
if (IncrementalCompilation.ENABLED) {
if (cache.saveFileToCache(target.getId(), outputFile)) {
if (cache.saveFileToCache(target.getId(), sourceFiles, outputFile)) {
significantChanges = true;
}
}
@@ -255,6 +272,15 @@ public class KotlinBuilder extends ModuleLevelBuilder {
return allCompiledFiles;
}
private static Set<ModuleBuildTarget> getProcessedTargetsWithRemovedFilesContainer(CompileContext context) {
Set<ModuleBuildTarget> set = PROCESSED_TARGETS_WITH_REMOVED_FILES.get(context);
if (set == null) {
set = new HashSet<ModuleBuildTarget>();
PROCESSED_TARGETS_WITH_REMOVED_FILES.set(context, set);
}
return set;
}
private static boolean hasKotlinFiles(@NotNull ModuleChunk chunk) {
boolean hasKotlinFiles = false;
for (ModuleBuildTarget target : chunk.getTargets()) {
@@ -59,7 +59,8 @@ public class KotlinBuilderModuleScriptGenerator {
public static File generateModuleDescription(
CompileContext context,
ModuleChunk chunk,
List<File> sourceFiles // ignored for non-incremental compilation
List<File> sourceFiles, // ignored for non-incremental compilation
boolean hasRemovedFiles
)
throws IOException
{
@@ -79,7 +80,7 @@ public class KotlinBuilderModuleScriptGenerator {
sourceFiles = new ArrayList<File>(KotlinSourceFileCollector.getAllKotlinSourceFiles(target));
}
if (sourceFiles.size() > 0) {
if (sourceFiles.size() > 0 || hasRemovedFiles) {
noSources = false;
if (logger.isEnabled()) {