Loading descriptors from incremental cache instead of package classes.

annotations/com/intellij/openapi/util/annotations.xml

@@ -6,4 +6,14 @@
   <item name='com.intellij.openapi.util.Key com.intellij.openapi.util.Key&lt;T&gt; create(java.lang.String)'>
     <annotation name='org.jetbrains.annotations.NotNull'/>
   </item>
+  <item name='com.intellij.openapi.util.Pair A getFirst()'>
+    <annotation name='kotlin.jvm.KotlinSignature'>
+      <val name="value" val="&quot;fun getFirst(): A&quot;"/>
+    </annotation>
+  </item>
+  <item name='com.intellij.openapi.util.Pair B getSecond()'>
+    <annotation name='kotlin.jvm.KotlinSignature'>
+      <val name="value" val="&quot;fun getSecond(): B&quot;"/>
+    </annotation>
+  </item>
 </root>

compiler/backend/src/org/jetbrains/jet/codegen/PackageCodegen.java

@@ -48,10 +48,9 @@ import org.jetbrains.jet.lang.resolve.BindingContext;
 import org.jetbrains.jet.lang.resolve.MemberComparator;
 import org.jetbrains.jet.lang.resolve.java.JvmAbi;
 import org.jetbrains.jet.lang.resolve.java.JvmAnnotationNames;
-import org.jetbrains.jet.lang.resolve.java.JvmClassName;
 import org.jetbrains.jet.lang.resolve.java.jvmSignature.JvmMethodSignature;
-import org.jetbrains.jet.lang.resolve.java.lazy.descriptors.LazyJavaPackageFragmentScope;
 import org.jetbrains.jet.lang.resolve.kotlin.PackagePartClassUtils;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalPackageFragmentProvider;
 import org.jetbrains.jet.lang.resolve.name.FqName;
 import org.jetbrains.org.objectweb.asm.AnnotationVisitor;
 import org.jetbrains.org.objectweb.asm.MethodVisitor;
@@ -113,7 +112,7 @@ public class PackageCodegen {
         // TODO rewrite it to something more robust when module system is implemented
         for (PackageFragmentDescriptor anotherFragment : packageFragment.getContainingDeclaration().getPackageFragmentProvider()
                 .getPackageFragments(packageFragment.getFqName())) {
-            if (anotherFragment.getMemberScope() instanceof LazyJavaPackageFragmentScope) {
+            if (anotherFragment instanceof IncrementalPackageFragmentProvider.IncrementalPackageFragment) {
                 return anotherFragment;
             }
         }

compiler/cli/cli-common/src/org/jetbrains/jet/cli/common/modules/ModuleDescription.java

@@ -18,6 +18,7 @@ package org.jetbrains.jet.cli.common.modules;
 
 import com.intellij.util.SmartList;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.List;
 
@@ -41,6 +42,9 @@ public interface ModuleDescription {
     @NotNull
     List<String> getAnnotationsRoots();
 
+    @Nullable
+    String getIncrementalCacheDir();
+
     class Impl implements ModuleDescription {
 
         private String name;
@@ -48,6 +52,7 @@ public interface ModuleDescription {
         private final List<String> sources = new SmartList<String>();
         private final List<String> classpath = new SmartList<String>();
         private final List<String> annotations = new SmartList<String>();
+        private String incrementalCacheDir;
 
         public void setName(String name) {
             this.name = name;
@@ -69,6 +74,10 @@ public interface ModuleDescription {
             annotations.add(path);
         }
 
+        public void setIncrementalCacheDir(String incrementalCacheDir) {
+            this.incrementalCacheDir = incrementalCacheDir;
+        }
+
         @NotNull
         @Override
         public String getModuleName() {
@@ -99,6 +108,12 @@ public interface ModuleDescription {
             return annotations;
         }
 
+        @Nullable
+        @Override
+        public String getIncrementalCacheDir() {
+            return incrementalCacheDir;
+        }
+
         @Override
         public String toString() {
             return name +

compiler/cli/cli-common/src/org/jetbrains/jet/cli/common/modules/ModuleXmlParser.java

@@ -40,6 +40,7 @@ public class ModuleXmlParser {
     public static final String MODULE = "module";
     public static final String NAME = "name";
     public static final String OUTPUT_DIR = "outputDir";
+    public static final String INCREMENTAL_CACHE = "incrementalCache";
     public static final String SOURCES = "sources";
     public static final String PATH = "path";
     public static final String CLASSPATH = "classpath";
@@ -117,7 +118,11 @@ public class ModuleXmlParser {
                 throw createError(qName);
             }
 
-            setCurrentState(new InsideModule(getAttribute(attributes, NAME, qName), getAttribute(attributes, OUTPUT_DIR, qName)));
+            setCurrentState(new InsideModule(
+                    getAttribute(attributes, NAME, qName),
+                    getAttribute(attributes, OUTPUT_DIR, qName),
+                    attributes.getValue(INCREMENTAL_CACHE)
+            ));
         }
 
         @Override
@@ -131,10 +136,11 @@ public class ModuleXmlParser {
     private class InsideModule extends DefaultHandler {
 
         private final ModuleDescription.Impl moduleDescription;
-        private InsideModule(String name, String outputDir) {
+        private InsideModule(String name, String outputDir, String incrementalCacheDir) {
             this.moduleDescription = new ModuleDescription.Impl();
             this.moduleDescription.setName(name);
             this.moduleDescription.setOutputDir(outputDir);
+            this.moduleDescription.setIncrementalCacheDir(incrementalCacheDir);
             result.add(moduleDescription);
         }
 

compiler/cli/src/org/jetbrains/jet/cli/jvm/JVMConfigurationKeys.java

@@ -38,4 +38,10 @@ public class JVMConfigurationKeys {
 
     public static final CompilerConfigurationKey<Boolean> ENABLE_INLINE =
             CompilerConfigurationKey.create("enable inline");
+
+    public static final CompilerConfigurationKey<File> INCREMENTAL_CACHE_BASE_DIR =
+            CompilerConfigurationKey.create("incremental cache base dir");
+
+    public static final CompilerConfigurationKey<List<String>> MODULE_IDS =
+            CompilerConfigurationKey.create("module id strings");
 }

compiler/cli/src/org/jetbrains/jet/cli/jvm/compiler/CompileEnvironmentUtil.java

@@ -324,7 +324,7 @@ public class CompileEnvironmentUtil {
         return result;
     }
 
-    private static class DescriptionToModuleAdapter implements Module {
+    static class DescriptionToModuleAdapter implements Module {
         private final ModuleDescription description;
 
         public DescriptionToModuleAdapter(ModuleDescription description) {
@@ -360,5 +360,10 @@ public class CompileEnvironmentUtil {
         public List<String> getAnnotationsRoots() {
             return description.getAnnotationsRoots();
         }
+
+        @Nullable
+        public String getIncrementalCacheDir() {
+            return description.getIncrementalCacheDir();
+        }
     }
 }

compiler/cli/src/org/jetbrains/jet/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.java

@@ -51,6 +51,7 @@ import org.jetbrains.jet.lang.resolve.BindingTrace;
 import org.jetbrains.jet.lang.resolve.ScriptNameUtil;
 import org.jetbrains.jet.lang.resolve.java.AnalyzerFacadeForJVM;
 import org.jetbrains.jet.lang.resolve.java.PackageClassUtils;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.CliSourcesMemberFilter;
 import org.jetbrains.jet.lang.resolve.name.FqName;
 import org.jetbrains.jet.plugin.MainFunctionDetector;
 import org.jetbrains.jet.utils.KotlinPaths;
@@ -162,7 +163,18 @@ public class KotlinToJVMBytecodeCompiler {
             for (String annotationsRoot : module.getAnnotationsRoots()) {
                 configuration.add(JVMConfigurationKeys.ANNOTATIONS_PATH_KEY, new File(annotationsRoot));
             }
+
+            configuration.add(JVMConfigurationKeys.MODULE_IDS, module.getModuleName());
         }
+
+        Module anyModule = chunk.get(0);
+        if (anyModule instanceof CompileEnvironmentUtil.DescriptionToModuleAdapter) {
+            String incrementalCacheDir = ((CompileEnvironmentUtil.DescriptionToModuleAdapter) anyModule).getIncrementalCacheDir();
+            if (incrementalCacheDir != null) {
+                configuration.put(JVMConfigurationKeys.INCREMENTAL_CACHE_BASE_DIR, new File(incrementalCacheDir));
+            }
+        }
+
         return configuration;
     }
 
@@ -286,7 +298,10 @@ public class KotlinToJVMBytecodeCompiler {
                                 sharedTrace,
                                 Predicates.<PsiFile>alwaysTrue(),
                                 sharedModule,
-                                new CliSourcesMemberFilter(environment));
+                                new CliSourcesMemberFilter(environment.getSourceFiles()),
+                                environment.getConfiguration().get(JVMConfigurationKeys.MODULE_IDS),
+                                environment.getConfiguration().get(JVMConfigurationKeys.INCREMENTAL_CACHE_BASE_DIR)
+                        );
                     }
                 }, environment.getSourceFiles()
         );

compiler/frontend.java/src/org/jetbrains/jet/di/InjectorForTopDownAnalyzerForJvm.java

@@ -27,6 +27,7 @@ import org.jetbrains.jet.lang.resolve.LazyTopDownAnalyzer;
 import org.jetbrains.jet.lang.resolve.MutablePackageFragmentProvider;
 import org.jetbrains.jet.descriptors.serialization.descriptors.MemberFilter;
 import org.jetbrains.jet.lang.resolve.java.JavaDescriptorResolver;
+import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializers;
 import org.jetbrains.jet.lang.resolve.java.JavaClassFinderImpl;
 import org.jetbrains.jet.lang.resolve.java.resolver.TraceBasedExternalSignatureResolver;
 import org.jetbrains.jet.lang.resolve.java.resolver.TraceBasedJavaResolverCache;
@@ -66,7 +67,6 @@ import org.jetbrains.jet.lang.resolve.TypeHierarchyResolver;
 import org.jetbrains.jet.lang.resolve.java.lazy.LazyJavaPackageFragmentProvider;
 import org.jetbrains.jet.lang.resolve.java.lazy.GlobalJavaResolverContext;
 import org.jetbrains.jet.lang.resolve.kotlin.DeserializedDescriptorResolver;
-import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializers;
 import org.jetbrains.jet.lang.resolve.kotlin.AnnotationDescriptorDeserializer;
 import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializersStorage;
 import org.jetbrains.jet.lang.resolve.kotlin.ConstantDescriptorDeserializer;
@@ -88,6 +88,7 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
     private final MutablePackageFragmentProvider mutablePackageFragmentProvider;
     private final MemberFilter memberFilter;
     private final JavaDescriptorResolver javaDescriptorResolver;
+    private final DescriptorDeserializers descriptorDeserializers;
     private final JavaClassFinderImpl javaClassFinder;
     private final TraceBasedExternalSignatureResolver traceBasedExternalSignatureResolver;
     private final TraceBasedJavaResolverCache traceBasedJavaResolverCache;
@@ -127,7 +128,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
     private final LazyJavaPackageFragmentProvider lazyJavaPackageFragmentProvider;
     private final GlobalJavaResolverContext globalJavaResolverContext;
     private final DeserializedDescriptorResolver deserializedDescriptorResolver;
-    private final DescriptorDeserializers descriptorDeserializers;
     private final AnnotationDescriptorDeserializer annotationDescriptorDeserializer;
     private final DescriptorDeserializersStorage descriptorDeserializersStorage;
     private final ConstantDescriptorDeserializer constantDescriptorDeserializer;
@@ -161,6 +161,7 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.globalJavaResolverContext = new GlobalJavaResolverContext(storageManager, javaClassFinder, virtualFileFinder, deserializedDescriptorResolver, psiBasedExternalAnnotationResolver, traceBasedExternalSignatureResolver, traceBasedErrorReporter, psiBasedMethodSignatureChecker, traceBasedJavaResolverCache, javaPropertyInitializerEvaluator);
         this.lazyJavaPackageFragmentProvider = new LazyJavaPackageFragmentProvider(globalJavaResolverContext, getModuleDescriptor());
         this.javaDescriptorResolver = new JavaDescriptorResolver(lazyJavaPackageFragmentProvider, getModuleDescriptor());
+        this.descriptorDeserializers = new DescriptorDeserializers();
         this.bodyResolver = new BodyResolver();
         this.annotationResolver = new AnnotationResolver();
         this.callResolver = new CallResolver();
@@ -189,7 +190,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.overloadResolver = new OverloadResolver();
         this.overrideResolver = new OverrideResolver();
         this.typeHierarchyResolver = new TypeHierarchyResolver();
-        this.descriptorDeserializers = new DescriptorDeserializers();
         this.annotationDescriptorDeserializer = new AnnotationDescriptorDeserializer();
         this.descriptorDeserializersStorage = new DescriptorDeserializersStorage(storageManager);
         this.constantDescriptorDeserializer = new ConstantDescriptorDeserializer();
@@ -212,6 +212,9 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         this.lazyTopDownAnalyzer.setOverrideResolver(overrideResolver);
         this.lazyTopDownAnalyzer.setTrace(bindingTrace);
 
+        this.descriptorDeserializers.setAnnotationDescriptorDeserializer(annotationDescriptorDeserializer);
+        this.descriptorDeserializers.setConstantDescriptorDeserializer(constantDescriptorDeserializer);
+
         javaClassFinder.setProject(project);
 
         traceBasedExternalSignatureResolver.setExternalAnnotationResolver(psiBasedExternalAnnotationResolver);
@@ -327,9 +330,6 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         deserializedDescriptorResolver.setMemberFilter(memberFilter);
         deserializedDescriptorResolver.setStorageManager(storageManager);
 
-        descriptorDeserializers.setAnnotationDescriptorDeserializer(annotationDescriptorDeserializer);
-        descriptorDeserializers.setConstantDescriptorDeserializer(constantDescriptorDeserializer);
-
         annotationDescriptorDeserializer.setClassResolver(javaDescriptorResolver);
         annotationDescriptorDeserializer.setErrorReporter(traceBasedErrorReporter);
         annotationDescriptorDeserializer.setKotlinClassFinder(virtualFileFinder);
@@ -367,4 +367,8 @@ public class InjectorForTopDownAnalyzerForJvm implements InjectorForTopDownAnaly
         return this.javaDescriptorResolver;
     }
     
+    public DescriptorDeserializers getDescriptorDeserializers() {
+        return this.descriptorDeserializers;
+    }
+    
 }

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/java/AnalyzerFacadeForJVM.java

@@ -32,14 +32,20 @@ import org.jetbrains.jet.di.InjectorForTopDownAnalyzerForJvm;
 import org.jetbrains.jet.lang.descriptors.DependencyKind;
 import org.jetbrains.jet.lang.descriptors.ModuleDescriptorImpl;
 import org.jetbrains.jet.lang.psi.JetFile;
-import org.jetbrains.jet.lang.resolve.*;
+import org.jetbrains.jet.lang.resolve.BindingTrace;
+import org.jetbrains.jet.lang.resolve.BindingTraceContext;
+import org.jetbrains.jet.lang.resolve.ImportPath;
+import org.jetbrains.jet.lang.resolve.TopDownAnalysisParameters;
 import org.jetbrains.jet.lang.resolve.java.mapping.JavaToKotlinClassMap;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalPackageFragmentProvider;
 import org.jetbrains.jet.lang.resolve.lazy.ResolveSession;
 import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactory;
 import org.jetbrains.jet.lang.resolve.lazy.declarations.DeclarationProviderFactoryService;
 import org.jetbrains.jet.lang.resolve.name.Name;
 import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns;
 
+import java.io.File;
 import java.util.Collection;
 import java.util.List;
 
@@ -124,7 +130,9 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
             BindingTrace trace,
             Predicate<PsiFile> filesToAnalyzeCompletely,
             ModuleDescriptorImpl module,
-            MemberFilter memberFilter
+            MemberFilter memberFilter,
+            List<String> moduleIds,
+            File incrementalCacheDir 
     ) {
         GlobalContext globalContext = ContextPackage.GlobalContext();
         TopDownAnalysisParameters topDownAnalysisParameters = TopDownAnalysisParameters.create(
@@ -139,6 +147,19 @@ public enum AnalyzerFacadeForJVM implements AnalyzerFacade {
                                                                                          memberFilter);
         try {
             module.addFragmentProvider(DependencyKind.BINARIES, injector.getJavaDescriptorResolver().getPackageFragmentProvider());
+
+            if (incrementalCacheDir != null && moduleIds != null) {
+                for (String moduleId : moduleIds) {
+                    module.addFragmentProvider(
+                            DependencyKind.SOURCES,
+                            new IncrementalPackageFragmentProvider(
+                                    files, module, globalContext.getStorageManager(), injector.getDescriptorDeserializers(),
+                                    new IncrementalCache(incrementalCacheDir), moduleId, injector.getJavaDescriptorResolver()
+                            )
+                    );
+                }
+            }
+
             injector.getTopDownAnalyzer().analyzeFiles(topDownAnalysisParameters, files);
             return AnalyzeExhaust.success(trace.getBindingContext(), module);
         }

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/VirtualFileKotlinClass.java

@@ -50,7 +50,7 @@ public class VirtualFileKotlinClass implements KotlinJvmBinaryClass {
     }
 
     @Nullable
-    private static Pair<JvmClassName, KotlinClassHeader> readClassNameAndHeader(@NotNull byte[] fileContents) {
+    public static Pair<JvmClassName, KotlinClassHeader> readClassNameAndHeader(@NotNull byte[] fileContents) {
         final ReadKotlinClassHeaderAnnotationVisitor readHeaderVisitor = new ReadKotlinClassHeaderAnnotationVisitor();
         final Ref<JvmClassName> classNameRef = Ref.create();
         new ClassReader(fileContents).accept(new ClassVisitor(ASM5) {

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/CliSourcesMemberFilter.kt

@@ -14,26 +14,25 @@
  * limitations under the License.
  */
 
-package org.jetbrains.jet.cli.jvm.compiler
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
 
+import org.jetbrains.jet.lang.psi.JetFile
 import org.jetbrains.jet.descriptors.serialization.descriptors.MemberFilter
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
 import org.jetbrains.jet.descriptors.serialization.ProtoBuf
-import org.jetbrains.jet.lang.resolve.kotlin.PackagePartClassUtils.getPackagePartInternalName
 import org.jetbrains.jet.descriptors.serialization.NameResolver
 import org.jetbrains.jet.descriptors.serialization.JavaProtoBuf
-import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
-import org.jetbrains.jet.lang.resolve.java.JvmClassName
-import org.jetbrains.jet.codegen.AsmUtil
 import org.jetbrains.jet.lang.resolve.kotlin.PackagePartClassUtils
+import org.jetbrains.jet.lang.resolve.java.JvmClassName
 
-public class CliSourcesMemberFilter(environment: JetCoreEnvironment): MemberFilter {
+public class CliSourcesMemberFilter(files: Collection<JetFile>): MemberFilter {
-    val packagePartClassNames = environment.getSourceFiles().map { PackagePartClassUtils.getPackagePartInternalName(it) }.toSet()
+    val packagePartClassNames = files.map { PackagePartClassUtils.getPackagePartInternalName(it) }.toSet()
 
     override fun acceptPackagePartClass(container: PackageFragmentDescriptor, member: ProtoBuf.Callable, nameResolver: NameResolver): Boolean {
         if (member.hasExtension(JavaProtoBuf.implClassName)) {
             val shortName = nameResolver.getName(member.getExtension(JavaProtoBuf.implClassName)!!)
             val fqName = container.fqName.child(shortName)
-            val internalName = AsmUtil.internalNameByFqNameWithoutInnerClasses(fqName)
+            val internalName = JvmClassName.byFqNameWithoutInnerClasses(fqName).getInternalName()
             return internalName !in packagePartClassNames
         }
         return true

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/IncrementalCache.kt

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
+
+import java.io.File
+import com.intellij.util.io.PersistentMap
+import com.intellij.util.io.PersistentHashMap
+import com.intellij.util.io.KeyDescriptor
+import java.io.DataOutput
+import com.intellij.util.io.IOUtil
+import java.io.DataInput
+import org.jetbrains.jet.lang.resolve.name.FqName
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.jet.lang.resolve.kotlin.VirtualFileKotlinClass
+import org.jetbrains.jet.lang.resolve.kotlin.header.KotlinClassHeader
+import org.jetbrains.jet.descriptors.serialization.BitEncoding
+import org.jetbrains.jet.utils.intellij.*
+
+public class IncrementalCache(baseDir: File) {
+    class object {
+        val PACKAGE_MAP: String = "package.tab"
+    }
+
+    private val packageFacadeData: PersistentMap<PackageFacadeId, ByteArray>
+
+    {
+        packageFacadeData = PersistentHashMap<PackageFacadeId, ByteArray>(File(baseDir, PACKAGE_MAP), object : KeyDescriptor<PackageFacadeId> {
+            override fun save(out: DataOutput, value: PackageFacadeId?) {
+                IOUtil.writeString(value!!.moduleId, out)
+                IOUtil.writeString(value.fqName.asString(), out)
+            }
+
+            override fun read(`in`: DataInput): PackageFacadeId {
+                val module = IOUtil.readString(`in`)!!
+                val fqName = FqName(IOUtil.readString(`in`)!!)
+                return PackageFacadeId(module, fqName)
+            }
+
+            override fun getHashCode(value: PackageFacadeId?): Int {
+                return value?.hashCode() ?: -1
+            }
+
+            override fun isEqual(val1: PackageFacadeId?, val2: PackageFacadeId?): Boolean {
+                return val1 == val2
+            }
+        }, object : DataExternalizer<ByteArray> {
+            override fun save(out: DataOutput, value: ByteArray?) {
+                out.writeInt(value!!.size)
+                out.write(value)
+            }
+
+            override fun read(`in`: DataInput): ByteArray {
+                val length = `in`.readInt()
+                val buf = ByteArray(length)
+                `in`.readFully(buf)
+                return buf
+            }
+        })
+    }
+
+    public fun saveFileToCache(moduleId: String, file: File) {
+        val classNameAndHeader = VirtualFileKotlinClass.readClassNameAndHeader(file.readBytes())
+        if (classNameAndHeader == null) return
+
+        val (className, header) = classNameAndHeader
+        if (header.kind == KotlinClassHeader.Kind.PACKAGE_FACADE) {
+            putPackageData(moduleId, className.getFqNameForClassNameWithoutDollars().parent(), BitEncoding.decodeBytes(header.annotationData!!))
+        }
+    }
+
+    public fun putPackageData(moduleId: String, fqName: FqName, data: ByteArray) {
+        packageFacadeData.put(PackageFacadeId(moduleId, fqName), data)
+    }
+
+    public fun getPackageData(moduleId: String, fqName: FqName): ByteArray? {
+        return packageFacadeData[PackageFacadeId(moduleId, fqName)]
+    }
+
+    public fun close() {
+        packageFacadeData.close()
+    }
+
+    private data class PackageFacadeId(val moduleId: String, val fqName: FqName) {
+    }
+}

compiler/frontend.java/src/org/jetbrains/jet/lang/resolve/kotlin/incremental/IncrementalPackageFragmentProvider.kt

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.lang.resolve.kotlin.incremental
+
+import org.jetbrains.jet.lang.resolve.name.FqName
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
+import org.jetbrains.jet.lang.psi.JetFile
+import com.intellij.util.containers.MultiMap
+import java.util.HashMap
+import org.jetbrains.jet.lang.descriptors.ModuleDescriptor
+import org.jetbrains.jet.lang.descriptors.PackageFragmentProvider
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptorImpl
+import org.jetbrains.jet.lang.resolve.scopes.JetScope
+import org.jetbrains.jet.storage.StorageManager
+import org.jetbrains.jet.descriptors.serialization.descriptors.DeserializedPackageMemberScope
+import org.jetbrains.jet.descriptors.serialization.descriptors.Deserializers
+import org.jetbrains.jet.descriptors.serialization.DescriptorFinder
+import org.jetbrains.jet.descriptors.serialization.JavaProtoBufUtil
+import org.jetbrains.jet.lang.resolve.java.JavaDescriptorResolver
+import org.jetbrains.jet.descriptors.serialization.ClassId
+import org.jetbrains.jet.lang.descriptors.ClassDescriptor
+import org.jetbrains.jet.lang.resolve.name.Name
+import org.jetbrains.jet.lang.resolve.kotlin.DeserializedResolverUtils
+import java.util.Collections
+import org.jetbrains.jet.utils.addToStdlib.singletonOrEmptyList
+import org.jetbrains.jet.storage.NotNullLazyValue
+
+public class IncrementalPackageFragmentProvider(
+        sourceFiles: Collection<JetFile>,
+        val module: ModuleDescriptor,
+        val storageManager: StorageManager,
+        val deserializers: Deserializers,
+        val incrementalCache: IncrementalCache,
+        val moduleId: String,
+        val javaDescriptorResolver: JavaDescriptorResolver
+
+) : PackageFragmentProvider {
+
+    val memberFilter = CliSourcesMemberFilter(sourceFiles)
+    val fqNameToSubFqNames = MultiMap<FqName, FqName>()
+    val fqNameToPackageFragment = HashMap<FqName, PackageFragmentDescriptor>()
+
+    ;{
+        fun createPackageFragment(fqName: FqName) {
+            if (fqNameToPackageFragment.containsKey(fqName)) {
+                return
+            }
+
+            if (!fqName.isRoot()) {
+                val parent = fqName.parent()
+                createPackageFragment(parent)
+                fqNameToSubFqNames.putValue(parent, fqName)
+            }
+
+            fqNameToPackageFragment[fqName] = IncrementalPackageFragment(fqName)
+        }
+
+        for (source in sourceFiles) {
+            createPackageFragment(source.getPackageFqName())
+        }
+    }
+
+    override fun getSubPackagesOf(fqName: FqName): Collection<FqName> {
+        return fqNameToSubFqNames[fqName].orEmpty()
+    }
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> {
+        return fqNameToPackageFragment[fqName].singletonOrEmptyList()
+    }
+
+
+    public inner class IncrementalPackageFragment(fqName: FqName) : PackageFragmentDescriptorImpl(module, fqName) {
+        val descriptorFinder = object : DescriptorFinder {
+            override fun findClass(classId: ClassId): ClassDescriptor? =
+                    javaDescriptorResolver.resolveClass(DeserializedResolverUtils.kotlinFqNameToJavaFqName(classId.asSingleFqName()))
+
+            override fun getClassNames(packageName: FqName): Collection<Name> {
+                return Collections.emptySet()
+            }
+        }
+
+        val _memberScope: NotNullLazyValue<JetScope> = storageManager.createLazyValue {
+            val packageDataBytes = incrementalCache.getPackageData(moduleId, fqName)
+            if (packageDataBytes == null) {
+                JetScope.EMPTY
+            }
+            else {
+                val packageData = JavaProtoBufUtil.readPackageDataFrom(packageDataBytes)
+                DeserializedPackageMemberScope(
+                        storageManager,
+                        this,
+                        deserializers,
+                        memberFilter,
+                        descriptorFinder,
+                        packageData
+                )
+            }
+
+        }
+
+        override fun getMemberScope(): JetScope {
+            return _memberScope()
+        }
+    }
+}
+

compiler/tests/org/jetbrains/jet/checkers/AbstractJetDiagnosticsTest.java

@@ -86,7 +86,9 @@ public abstract class AbstractJetDiagnosticsTest extends BaseDiagnosticsTest {
                     moduleTrace,
                     Predicates.<PsiFile>alwaysTrue(),
                     module == null ? support.getModule() : module,
-                    MemberFilter.ALWAYS_TRUE
+                    MemberFilter.ALWAYS_TRUE,
+                    null,
+                    null
             );
         }
 

compiler/tests/org/jetbrains/jet/lang/resolve/lazy/JvmResolveUtil.java

@@ -76,6 +76,6 @@ public class JvmResolveUtil {
 
         return AnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(project, files, bindingTraceContext, filesToAnalyzeCompletely,
                                                                     AnalyzerFacadeForJVM.createJavaModule("<module>"),
-                                                                    MemberFilter.ALWAYS_TRUE);
+                                                                    MemberFilter.ALWAYS_TRUE, null, null);
     }
 }

compiler/util/src/org/jetbrains/jet/utils/addToStdlib.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.jet.utils.addToStdlib
 
 import java.util.HashMap
+import java.util.Collections
 
 fun <K, V> Map<K, V>.filterKeys(predicate: (K)->Boolean): Map<K, V> {
     val result = HashMap<K, V>()
@@ -27,3 +28,7 @@ fun <K, V> Map<K, V>.filterKeys(predicate: (K)->Boolean): Map<K, V> {
     }
     return result
 }
+
+fun <T: Any> T?.singletonOrEmptyList(): List<T> = if (this != null) Collections.singletonList(this) else Collections.emptyList()
+
+fun <T: Any> T?.singletonOrEmptySet(): Set<T> = if (this != null) Collections.singleton(this) else Collections.emptySet()

compiler/util/src/org/jetbrains/jet/utils/intellijUtil.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.utils.intellij
+
+import com.intellij.openapi.util.Pair
+
+public fun <A> Pair<A, *>.component1(): A = getFirst()
+
+public fun <B> Pair<*, B>.component2(): B = getSecond()

core/serialization.java/src/org/jetbrains/jet/descriptors/serialization/JavaProtoBufUtil.java

@@ -31,12 +31,17 @@ public class JavaProtoBufUtil {
     }
 
     @NotNull
-    public static ClassData readClassDataFrom(@NotNull String[] data) {
+    public static ClassData readClassDataFrom(@NotNull String[] encodedData) {
-        return ClassData.read(BitEncoding.decodeBytes(data), getExtensionRegistry());
+        return ClassData.read(BitEncoding.decodeBytes(encodedData), getExtensionRegistry());
     }
 
     @NotNull
-    public static PackageData readPackageDataFrom(@NotNull String[] data) {
+    public static PackageData readPackageDataFrom(@NotNull String[] encodedData) {
-        return PackageData.read(BitEncoding.decodeBytes(data), getExtensionRegistry());
+        return readPackageDataFrom(BitEncoding.decodeBytes(encodedData));
+    }
+
+    @NotNull
+    public static PackageData readPackageDataFrom(@NotNull byte[] data) {
+        return PackageData.read(data, getExtensionRegistry());
     }
 }

generators/src/org/jetbrains/jet/generators/injectors/GenerateInjectors.kt

@@ -36,6 +36,7 @@ import org.jetbrains.jet.lang.types.expressions.ExpressionTypingUtils
 import org.jetbrains.jet.descriptors.serialization.descriptors.MemberFilter
 import org.jetbrains.jet.lang.resolve.calls.CallResolver
 import org.jetbrains.jet.lang.resolve.java.structure.impl.JavaPropertyInitializerEvaluatorImpl
+import org.jetbrains.jet.lang.resolve.kotlin.DescriptorDeserializers
 
 // NOTE: After making changes, you need to re-generate the injectors.
 //       To do that, you can run main in this file.
@@ -100,6 +101,7 @@ private fun generatorForTopDownAnalyzerForJvm() =
             parameter(javaClass<MemberFilter>())
 
             publicField(javaClass<JavaDescriptorResolver>())
+            publicField(javaClass<DescriptorDeserializers>())
 
             fields(
                     javaClass<JavaClassFinderImpl>(),

ide-compiler-runner/src/org/jetbrains/jet/compiler/runner/KotlinModuleDescriptionBuilder.java

@@ -25,10 +25,12 @@ public interface KotlinModuleDescriptionBuilder {
     KotlinModuleDescriptionBuilder addModule(
             String moduleName,
             String outputDir,
+            String incrementalCacheDir,
             DependencyProvider dependencyProvider,
             List<File> sourceFiles,
             boolean tests,
-            Set<File> directoriesToFilterOut);
+            Set<File> directoriesToFilterOut
+    );
 
     CharSequence asText();
 

ide-compiler-runner/src/org/jetbrains/jet/compiler/runner/KotlinModuleScriptBuilderFactory.java

@@ -54,6 +54,7 @@ public class KotlinModuleScriptBuilderFactory implements KotlinModuleDescription
         public KotlinModuleDescriptionBuilder addModule(
                 String moduleName,
                 String outputDir,
+                String incrementalCacheDir,
                 DependencyProvider dependencyProvider,
                 List<File> sourceFiles,
                 boolean tests,

ide-compiler-runner/src/org/jetbrains/jet/compiler/runner/KotlinModuleXmlBuilderFactory.java

@@ -58,6 +58,7 @@ public class KotlinModuleXmlBuilderFactory implements KotlinModuleDescriptionBui
         public KotlinModuleDescriptionBuilder addModule(
                 String moduleName,
                 String outputDir,
+                String incrementalCacheDir,
                 DependencyProvider dependencyProvider,
                 List<File> sourceFiles,
                 boolean tests,
@@ -74,7 +75,9 @@ public class KotlinModuleXmlBuilderFactory implements KotlinModuleDescriptionBui
 
             p.println("<", MODULE, " ",
                       NAME, "=\"", escapeXml(moduleName), "\" ",
-                      OUTPUT_DIR, "=\"", getEscapedPath(new File(outputDir)), "\">");
+                      OUTPUT_DIR, "=\"", getEscapedPath(new File(outputDir)), "\"",
+                      incrementalCacheDir != null ? " " + INCREMENTAL_CACHE + "=\"" + getEscapedPath(new File(incrementalCacheDir)) + "\" " : "",
+                      ">");
             p.pushIndent();
 
             for (File sourceFile : sourceFiles) {

idea/src/org/jetbrains/jet/plugin/compiler/K2JvmTranslatingCompiler.java

@@ -171,6 +171,7 @@ public class K2JvmTranslatingCompiler implements TranslatingCompiler {
         CharSequence script = KotlinModuleScriptBuilderFactory.INSTANCE.create().addModule(
                 moduleName,
                 moduleOutputDirectory.getAbsolutePath(),
+                null,
                 getDependencyProvider(chunk, tests, mainOutput),
                 sourceFiles,
                 tests,

idea/tests/org/jetbrains/jet/modules/xml/KotlinModuleXmlGeneratorTest.java

@@ -31,6 +31,7 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
         String actual = KotlinModuleXmlBuilderFactory.INSTANCE.create().addModule(
                 "name",
                 "output",
+                null,
                 new KotlinModuleDescriptionBuilder.DependencyProvider() {
                     @Override
                     public void processClassPath(@NotNull KotlinModuleDescriptionBuilder.DependencyProcessor processor) {
@@ -48,6 +49,7 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
         String actual = KotlinModuleXmlBuilderFactory.INSTANCE.create().addModule(
                 "name",
                 "output",
+                null,
                 new KotlinModuleDescriptionBuilder.DependencyProvider() {
                     @Override
                     public void processClassPath(@NotNull KotlinModuleDescriptionBuilder.DependencyProcessor processor) {
@@ -66,6 +68,7 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
         builder.addModule(
                 "name",
                 "output",
+                null,
                 new KotlinModuleDescriptionBuilder.DependencyProvider() {
                     @Override
                     public void processClassPath(@NotNull KotlinModuleDescriptionBuilder.DependencyProcessor processor) {
@@ -79,6 +82,7 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
         builder.addModule(
                 "name2",
                 "output2",
+                null,
                 new KotlinModuleDescriptionBuilder.DependencyProvider() {
                     @Override
                     public void processClassPath(@NotNull KotlinModuleDescriptionBuilder.DependencyProcessor processor) {

jps-plugin/jps-plugin.iml

@@ -15,8 +15,8 @@
     <orderEntry type="library" scope="TEST" name="jps-test" level="project" />
     <orderEntry type="library" scope="TEST" name="idea-full" level="project" />
     <orderEntry type="module" module-name="backend" scope="TEST" />
-    <orderEntry type="module" module-name="descriptors" scope="TEST" />
+    <orderEntry type="module" module-name="descriptors" />
-    <orderEntry type="module" module-name="frontend.java" scope="TEST" />
+    <orderEntry type="module" module-name="frontend.java" />
     <orderEntry type="module" module-name="descriptor.loader.java" scope="TEST" />
     <orderEntry type="module" module-name="serialization.java" scope="TEST" />
     <orderEntry type="module" module-name="serialization" scope="TEST" />

jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java

@@ -37,6 +37,7 @@ import org.jetbrains.jet.compiler.runner.OutputItemsCollectorImpl;
 import org.jetbrains.jet.compiler.runner.SimpleOutputItem;
 import org.jetbrains.jet.config.IncrementalCompilation;
 import org.jetbrains.jet.jps.JpsKotlinCompilerSettings;
+import org.jetbrains.jet.lang.resolve.kotlin.incremental.IncrementalCache;
 import org.jetbrains.jet.utils.PathUtil;
 import org.jetbrains.jps.ModuleChunk;
 import org.jetbrains.jps.builders.BuildTarget;
@@ -194,38 +195,47 @@ public class KotlinBuilder extends ModuleLevelBuilder {
             }
         }
 
-        for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
+        IncrementalCache cache = new IncrementalCache(KotlinBuilderModuleScriptGenerator.getIncrementalCacheDir(context));
-            if (IncrementalCompilation.ENABLED) {
+
-                // TODO this is a hack: we don't remove
+        try {
-                if (outputItem.getOutputFile().getName().endsWith("Package.class")) {
+            for (SimpleOutputItem outputItem : outputItemCollector.getOutputs()) {
-                    continue;
+                BuildTarget<?> target = null;
+                Collection<File> sourceFiles = outputItem.getSourceFiles();
+                if (sourceFiles != null && !sourceFiles.isEmpty()) {
+                    target = sourceToTarget.get(sourceFiles.iterator().next());
                 }
+                else {
+                    messageCollector.report(EXCEPTION, "KotlinBuilder: outputItem.sourceFiles is null or empty, outputItem = " + outputItem, NO_LOCATION);
+                }
+
+                if (target == null) {
+                    target = representativeTarget;
+                }
+
+                File outputFile = outputItem.getOutputFile();
+
+                if (IncrementalCompilation.ENABLED) {
+                    cache.saveFileToCache(target.getId(), outputFile);
+                }
+
+                outputConsumer.registerOutputFile(target, outputFile, paths(sourceFiles));
             }
-            BuildTarget<?> target = null;
+
-            Collection<File> sourceFiles = outputItem.getSourceFiles();
+            if (IncrementalCompilation.ENABLED) {
-            if (sourceFiles != null && !sourceFiles.isEmpty()) {
+                // TODO should mark dependencies as dirty, as well
-                target = sourceToTarget.get(sourceFiles.iterator().next());
+                FSOperations.markDirty(context, chunk, new FileFilter() {
+                    @Override
+                    public boolean accept(@NotNull File file) {
+                        return !allCompiledFiles.contains(file);
+                    }
+                });
+                return ExitCode.ADDITIONAL_PASS_REQUIRED;
             }
             else {
-                messageCollector.report(EXCEPTION, "KotlinBuilder: outputItem.sourceFiles is null or empty, outputItem = " + outputItem, NO_LOCATION);
+                return ExitCode.OK;
             }
-
+        } finally {
-            outputConsumer.registerOutputFile(target != null ? target : representativeTarget, outputItem.getOutputFile(),
+            cache.close();
-                                              paths(sourceFiles));
-        }
-
-        if (IncrementalCompilation.ENABLED) {
-            // TODO should mark dependencies as dirty, as well
-            FSOperations.markDirty(context, chunk, new FileFilter() {
-                @Override
-                public boolean accept(@NotNull File file) {
-                    return !allCompiledFiles.contains(file);
-                }
-            });
-            return ExitCode.ADDITIONAL_PASS_REQUIRED;
-        }
-        else {
-            return ExitCode.OK;
         }
     }
 

jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java

@@ -90,6 +90,7 @@ public class KotlinBuilderModuleScriptGenerator {
             builder.addModule(
                     target.getId(),
                     outputDir.getAbsolutePath(),
+                    getIncrementalCacheDir(context).getAbsolutePath(),
                     getKotlinModuleDependencies(context, target),
                     sourceFiles,
                     target.isTests(),
@@ -109,6 +110,10 @@ public class KotlinBuilderModuleScriptGenerator {
         return scriptFile;
     }
 
+    public static File getIncrementalCacheDir(CompileContext context) {
+        return new File(context.getProjectDescriptor().dataManager.getDataPaths().getDataStorageRoot(), "kotlin");
+    }
+
     @NotNull
     private static File getOutputDir(@NotNull ModuleBuildTarget target) {
         File outputDir = target.getOutputDir();

jps-plugin/testData/incremental/topLevelMembersInTwoPackages/build.log

@@ -1,11 +1,13 @@
 Cleaning output files:
 out/production/module/test/TestPackage-a-*.class
+out/production/module/test/TestPackage.class
 End of files
 Compiling files:
 src/a.kt
 End of files
 Cleaning output files:
 out/production/module/test/TestPackage-b-*.class
+out/production/module/test/TestPackage.class
 End of files
 Compiling files:
 src/b.kt