Loading descriptors from incremental cache instead of package classes.

compiler/frontend.java/src/org/jetbrains/jet/di/InjectorForTopDownAnalyzerForJvm.java

@@ -27,6 +27,7 @@ import org.jetbrains.jet.lang.resolve.LazyTopDownAnalyzer;
 import org.jetbrains.jet.lang.resolve.MutablePackageFragmentProvider;
 import org.jetbrains.jet.descriptors.serialization.descriptors.MemberFilter;
 import org.jetbrains.jet.lang.resolve.java.JavaDescriptorResolver;
+import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializers;
 import org.jetbrains.jet.lang.resolve.java.JavaClassFinderImpl;
 import org.jetbrains.jet.lang.resolve.java.resolver.TraceBasedExternalSignatureResolver;
 import org.jetbrains.jet.lang.resolve.java.resolver.TraceBasedJavaResolverCache;
@@ -66,7 +67,6 @@ import org.jetbrains.jet.lang.resolve.TypeHierarchyResolver;
 import org.jetbrains.jet.lang.resolve.java.lazy.LazyJavaPackageFragmentProvider;
 import org.jetbrains.jet.lang.resolve.java.lazy.GlobalJavaResolverContext;
 import org.jetbrains.jet.lang.resolve.kotlin.DeserializedDescriptorResolver;
-import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializers;
 import org.jetbrains.jet.lang.resolve.kotlin.AnnotationDescriptorDeserializer;
 import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializersStorage;
 import org.jetbrains.jet.lang.resolve.kotlin.ConstantDescriptorDeserializer;
@@ -88,6 +88,7 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
     private final MutablePackageFragmentProvider mutablePackageFragmentProvider;
     private final MemberFilter memberFilter;
     private final JavaDescriptorResolver javaDescriptorResolver;
+    private final DescriptorDeserializers descriptorDeserializers;
     private final JavaClassFinderImpl javaClassFinder;
     private final TraceBasedExternalSignatureResolver traceBasedExternalSignatureResolver;
     private final TraceBasedJavaResolverCache traceBasedJavaResolverCache;
@@ -127,7 +128,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
     private final LazyJavaPackageFragmentProvider lazyJavaPackageFragmentProvider;
     private final GlobalJavaResolverContext globalJavaResolverContext;
     private final DeserializedDescriptorResolver deserializedDescriptorResolver;
-    private final DescriptorDeserializers descriptorDeserializers;
     private final AnnotationDescriptorDeserializer annotationDescriptorDeserializer;
     private final DescriptorDeserializersStorage descriptorDeserializersStorage;
     private final ConstantDescriptorDeserializer constantDescriptorDeserializer;
@@ -161,6 +161,7 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.globalJavaResolverContext = new GlobalJavaResolverContext(storageManager, javaClassFinder, virtualFileFinder, deserializedDescriptorResolver, psiBasedExternalAnnotationResolver, traceBasedExternalSignatureResolver, traceBasedErrorReporter, psiBasedMethodSignatureChecker, traceBasedJavaResolverCache, javaPropertyInitializerEvaluator);
         this.lazyJavaPackageFragmentProvider = new LazyJavaPackageFragmentProvider(globalJavaResolverContext, getModuleDescriptor());
         this.javaDescriptorResolver = new JavaDescriptorResolver(lazyJavaPackageFragmentProvider, getModuleDescriptor());
+        this.descriptorDeserializers = new DescriptorDeserializers();
         this.bodyResolver = new BodyResolver();
         this.annotationResolver = new AnnotationResolver();
         this.callResolver = new CallResolver();
@@ -189,7 +190,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.overloadResolver = new OverloadResolver();
         this.overrideResolver = new OverrideResolver();
         this.typeHierarchyResolver = new TypeHierarchyResolver();
-        this.descriptorDeserializers = new DescriptorDeserializers();
         this.annotationDescriptorDeserializer = new AnnotationDescriptorDeserializer();
         this.descriptorDeserializersStorage = new DescriptorDeserializersStorage(storageManager);
         this.constantDescriptorDeserializer = new ConstantDescriptorDeserializer();
@@ -212,6 +212,9 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.lazyTopDownAnalyzer.setOverrideResolver(overrideResolver);
         this.lazyTopDownAnalyzer.setTrace(bindingTrace);
 
+        this.descriptorDeserializers.setAnnotationDescriptorDeserializer(annotationDescriptorDeserializer);
+        this.descriptorDeserializers.setConstantDescriptorDeserializer(constantDescriptorDeserializer);
+
         javaClassFinder.setProject(project);
 
         traceBasedExternalSignatureResolver.setExternalAnnotationResolver(psiBasedExternalAnnotationResolver);
@@ -327,9 +330,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         deserializedDescriptorResolver.setMemberFilter(memberFilter);
         deserializedDescriptorResolver.setStorageManager(storageManager);
 
-        descriptorDeserializers.setAnnotationDescriptorDeserializer(annotationDescriptorDeserializer);
-        descriptorDeserializers.setConstantDescriptorDeserializer(constantDescriptorDeserializer);
-
         annotationDescriptorDeserializer.setClassResolver(javaDescriptorResolver);
         annotationDescriptorDeserializer.setErrorReporter(traceBasedErrorReporter);
         annotationDescriptorDeserializer.setKotlinClassFinder(virtualFileFinder);
@@ -367,4 +367,8 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         return this.javaDescriptorResolver;
     }
     
+    public DescriptorDeserializers getDescriptorDeserializers() {
+        return this.descriptorDeserializers;
+    }
+    
 }

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/java/AnalyzerFacadeForJVM.java

@@ -32,14 +32,20 @@ import org.jetbrains.jet.di.InjectorForTopDownAnalyzerForJvm;
 import org.jetbrains.jet.lang.descriptors.DependencyKind;
 import org.jetbrains.jet.lang.descriptors.ModuleDescriptorImpl;
 import org.jetbrains.jet.lang.psi.JetFile;
-import org.jetbrains.jet.lang.resolve.*;
+import org.jetbrains.jet.lang.resolve.BindingTrace;
+import org.jetbrains.jet.lang.resolve.BindingTraceContext;
+import org.jetbrains.jet.lang.resolve.ImportPath;
+import org.jetbrains.jet.lang.resolve.TopDownAnalysisParameters;
 import org.jetbrains.jet.lang.resolve.java.mapping.JavaToKotlinClassMap;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalPackageFragmentProvider;
 import org.jetbrains.jet.lang.resolve.lazy.ResolveSession;
 import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactory;
 import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactoryService;
 import org.jetbrains.jet.lang.resolve.name.Name;
 import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns;
 
+import java.io.File;
 import java.util.Collection;
 import java.util.List;
 
@@ -124,7 +130,9 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
             BindingTrace trace,
             Predicate<PsiFile> filesToAnalyzeCompletely,
             ModuleDescriptorImpl module,
-            MemberFilter memberFilter
+            MemberFilter memberFilter,
+            List<String> moduleIds,
+            File incrementalCacheDir 
     ) {
         GlobalContext globalContext = ContextPackage.GlobalContext();
         TopDownAnalysisParameters topDownAnalysisParameters = TopDownAnalysisParameters.create(
@@ -139,6 +147,19 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
                                                                                          memberFilter);
         try {
             module.addFragmentProvider(DependencyKind.BINARIES, injector.getJavaDescriptorResolver().getPackageFragmentProvider());
+
+            if (incrementalCacheDir != null && moduleIds != null) {
+                for (String moduleId : moduleIds) {
+                    module.addFragmentProvider(
+                            DependencyKind.SOURCES,
+                            new IncrementalPackageFragmentProvider(
+                                    files, module, globalContext.getStorageManager(), injector.getDescriptorDeserializers(),
+                                    new IncrementalCache(incrementalCacheDir), moduleId, injector.getJavaDescriptorResolver()
+                            )
+                    );
+                }
+            }
+
             injector.getTopDownAnalyzer().analyzeFiles(topDownAnalysisParameters, files);
             return AnalyzeExhaust.success(trace.getBindingContext(), module);
         }

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/VirtualFileKotlinClass.java

@@ -50,7 +50,7 @@ public class VirtualFileKotlinClass implements KotlinJvmBinaryClass {
     }
 
     @Nullable
-    private static Pair<JvmClassName, KotlinClassHeader> readClassNameAndHeader(@NotNull byte[] fileContents) {
+    public static Pair<JvmClassName, KotlinClassHeader> readClassNameAndHeader(@NotNull byte[] fileContents) {
         final ReadKotlinClassHeaderAnnotationVisitor readHeaderVisitor = new ReadKotlinClassHeaderAnnotationVisitor();
         final Ref<JvmClassName> classNameRef = Ref.create();
         new ClassReader(fileContents).accept(new ClassVisitor(ASM5) {

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/CliSourcesMemberFilter.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
+
+import org.jetbrains.jet.lang.psi.JetFile
+import org.jetbrains.jet.descriptors.serialization.descriptors.MemberFilter
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
+import org.jetbrains.jet.descriptors.serialization.ProtoBuf
+import org.jetbrains.jet.descriptors.serialization.NameResolver
+import org.jetbrains.jet.descriptors.serialization.JavaProtoBuf
+import org.jetbrains.jet.lang.resolve.kotlin.PackagePartClassUtils
+import org.jetbrains.jet.lang.resolve.java.JvmClassName
+
+public class CliSourcesMemberFilter(files: Collection<JetFile>): MemberFilter {
+    val packagePartClassNames = files.map { PackagePartClassUtils.getPackagePartInternalName(it) }.toSet()
+
+    override fun acceptPackagePartClass(container: PackageFragmentDescriptor, member: ProtoBuf.Callable, nameResolver: NameResolver): Boolean {
+        if (member.hasExtension(JavaProtoBuf.implClassName)) {
+            val shortName = nameResolver.getName(member.getExtension(JavaProtoBuf.implClassName)!!)
+            val fqName = container.fqName.child(shortName)
+            val internalName = JvmClassName.byFqNameWithoutInnerClasses(fqName).getInternalName()
+            return internalName !in packagePartClassNames
+        }
+        return true
+    }
+}

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/IncrementalCache.kt

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
+
+import java.io.File
+import com.intellij.util.io.PersistentMap
+import com.intellij.util.io.PersistentHashMap
+import com.intellij.util.io.KeyDescriptor
+import java.io.DataOutput
+import com.intellij.util.io.IOUtil
+import java.io.DataInput
+import org.jetbrains.jet.lang.resolve.name.FqName
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.jet.lang.resolve.kotlin.VirtualFileKotlinClass
+import org.jetbrains.jet.lang.resolve.kotlin.header.KotlinClassHeader
+import org.jetbrains.jet.descriptors.serialization.BitEncoding
+import org.jetbrains.jet.utils.intellij.*
+
+public class IncrementalCache(baseDir: File) {
+    class object {
+        val PACKAGE_MAP: String = "package.tab"
+    }
+
+    private val packageFacadeData: PersistentMap<PackageFacadeId, ByteArray>
+
+    {
+        packageFacadeData = PersistentHashMap<PackageFacadeId, ByteArray>(File(baseDir, PACKAGE_MAP), object : KeyDescriptor<PackageFacadeId> {
+            override fun save(out: DataOutput, value: PackageFacadeId?) {
+                IOUtil.writeString(value!!.moduleId, out)
+                IOUtil.writeString(value.fqName.asString(), out)
+            }
+
+            override fun read(`in`: DataInput): PackageFacadeId {
+                val module = IOUtil.readString(`in`)!!
+                val fqName = FqName(IOUtil.readString(`in`)!!)
+                return PackageFacadeId(module, fqName)
+            }
+
+            override fun getHashCode(value: PackageFacadeId?): Int {
+                return value?.hashCode() ?: -1
+            }
+
+            override fun isEqual(val1: PackageFacadeId?, val2: PackageFacadeId?): Boolean {
+                return val1 == val2
+            }
+        }, object : DataExternalizer<ByteArray> {
+            override fun save(out: DataOutput, value: ByteArray?) {
+                out.writeInt(value!!.size)
+                out.write(value)
+            }
+
+            override fun read(`in`: DataInput): ByteArray {
+                val length = `in`.readInt()
+                val buf = ByteArray(length)
+                `in`.readFully(buf)
+                return buf
+            }
+        })
+    }
+
+    public fun saveFileToCache(moduleId: String, file: File) {
+        val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(file.readBytes())
+        if (classNameAndHeader == null) return
+
+        val (className, header) = classNameAndHeader
+        if (header.kind == KotlinClassHeader.Kind.PACKAGE_FACADE) {
+            putPackageData(moduleId, className.getFqNameForClassNameWithoutDollars().parent(), BitEncoding.decodeBytes(header.annotationData!!))
+        }
+    }
+
+    public fun putPackageData(moduleId: String, fqName: FqName, data: ByteArray) {
+        packageFacadeData.put(PackageFacadeId(moduleId, fqName), data)
+    }
+
+    public fun getPackageData(moduleId: String, fqName: FqName): ByteArray? {
+        return packageFacadeData[PackageFacadeId(moduleId, fqName)]
+    }
+
+    public fun close() {
+        packageFacadeData.close()
+    }
+
+    private data class PackageFacadeId(val moduleId: String, val fqName: FqName) {
+    }
+}

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/IncrementalPackageFragmentProvider.kt

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
+
+import org.jetbrains.jet.lang.resolve.name.FqName
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
+import org.jetbrains.jet.lang.psi.JetFile
+import com.intellij.util.containers.MultiMap
+import java.util.HashMap
+import org.jetbrains.jet.lang.descriptors.ModuleDescriptor
+import org.jetbrains.jet.lang.descriptors.PackageFragmentProvider
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptorImpl
+import org.jetbrains.jet.lang.resolve.scopes.JetScope
+import org.jetbrains.jet.storage.StorageManager
+import org.jetbrains.jet.descriptors.serialization.descriptors.DeserializedPackageMemberScope
+import org.jetbrains.jet.descriptors.serialization.descriptors.Deserializers
+import org.jetbrains.jet.descriptors.serialization.DescriptorFinder
+import org.jetbrains.jet.descriptors.serialization.JavaProtoBufUtil
+import org.jetbrains.jet.lang.resolve.java.JavaDescriptorResolver
+import org.jetbrains.jet.descriptors.serialization.ClassId
+import org.jetbrains.jet.lang.descriptors.ClassDescriptor
+import org.jetbrains.jet.lang.resolve.name.Name
+import org.jetbrains.jet.lang.resolve.kotlin.DeserializedResolverUtils
+import java.util.Collections
+import org.jetbrains.jet.utils.addToStdlib.singletonOrEmptyList
+import org.jetbrains.jet.storage.NotNullLazyValue
+
+public class IncrementalPackageFragmentProvider(
+        sourceFiles: Collection<JetFile>,
+        val module: ModuleDescriptor,
+        val storageManager: StorageManager,
+        val deserializers: Deserializers,
+        val incrementalCache: IncrementalCache,
+        val moduleId: String,
+        val javaDescriptorResolver: JavaDescriptorResolver
+
+) : PackageFragmentProvider {
+
+    val memberFilter = CliSourcesMemberFilter(sourceFiles)
+    val fqNameToSubFqNames = MultiMap<FqName, FqName>()
+    val fqNameToPackageFragment = HashMap<FqName, PackageFragmentDescriptor>()
+
+    ;{
+        fun createPackageFragment(fqName: FqName) {
+            if (fqNameToPackageFragment.containsKey(fqName)) {
+                return
+            }
+
+            if (!fqName.isRoot()) {
+                val parent = fqName.parent()
+                createPackageFragment(parent)
+                fqNameToSubFqNames.putValue(parent, fqName)
+            }
+
+            fqNameToPackageFragment[fqName] = IncrementalPackageFragment(fqName)
+        }
+
+        for (source in sourceFiles) {
+            createPackageFragment(source.getPackageFqName())
+        }
+    }
+
+    override fun getSubPackagesOf(fqName: FqName): Collection<FqName> {
+        return fqNameToSubFqNames[fqName].orEmpty()
+    }
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> {
+        return fqNameToPackageFragment[fqName].singletonOrEmptyList()
+    }
+
+
+    public inner class IncrementalPackageFragment(fqName: FqName) : PackageFragmentDescriptorImpl(module, fqName) {
+        val descriptorFinder = object : DescriptorFinder {
+            override fun findClass(classId: ClassId): ClassDescriptor? =
+                    javaDescriptorResolver.resolveClass(DeserializedResolverUtils.kotlinFqNameToJavaFqName(classId.asSingleFqName()))
+
+            override fun getClassNames(packageName: FqName): Collection<Name> {
+                return Collections.emptySet()
+            }
+        }
+
+        val _memberScope: NotNullLazyValue<JetScope> = storageManager.createLazyValue {
+            val packageDataBytes = incrementalCache.getPackageData(moduleId, fqName)
+            if (packageDataBytes == null) {
+                JetScope.EMPTY
+            }
+            else {
+                val packageData = JavaProtoBufUtil.readPackageDataFrom(packageDataBytes)
+                DeserializedPackageMemberScope(
+                        storageManager,
+                        this,
+                        deserializers,
+                        memberFilter,
+                        descriptorFinder,
+                        packageData
+                )
+            }
+
+        }
+
+        override fun getMemberScope(): JetScope {
+            return _memberScope()
+        }
+    }
+}
+