Refine dirty files computation in case of Java source changes

#KT-17621 In Progress
2017-11-30 15:24:52 +03:00
parent 26393d738f
commit ae6421476d
66 changed files with 634 additions and 140 deletions
@@ -29,6 +29,7 @@ import org.jetbrains.kotlin.load.kotlin.ModuleMapping
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
@@ -54,6 +55,7 @@ open class IncrementalJvmCache(
        private val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
        private val INLINE_FUNCTIONS = "inline-functions"
        private val INTERNAL_NAME_TO_SOURCE = "internal-name-to-source"
+        private val JAVA_SOURCES_PROTO_MAP = "java-sources-proto-map"

        private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
    }
@@ -68,6 +70,7 @@ open class IncrementalJvmCache(
    private val inlineFunctionsMap = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile))
    // todo: try to use internal names only?
    private val internalNameToSource = registerMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile))
+    private val javaSourcesProtoMap = registerMap(JavaSourcesProtoMap(JAVA_SOURCES_PROTO_MAP.storageFile))

    private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory" } }

@@ -169,6 +172,28 @@ open class IncrementalJvmCache(
        }
    }

+    fun saveJavaClassProto(source: File, serializedJavaClass: SerializedJavaClass, collector: ChangesCollector) {
+        val jvmClassName = JvmClassName.byClassId(serializedJavaClass.classId)
+        javaSourcesProtoMap.process(jvmClassName, serializedJavaClass, collector)
+        sourceToClassesMap.add(source, jvmClassName)
+        val (proto, nameResolver) = serializedJavaClass.toProtoData()
+        addToClassStorage(proto, nameResolver, source)
+
+        dirtyOutputClassesMap.notDirty(jvmClassName.internalName)
+    }
+
+    fun getObsoleteJavaClasses(): Collection<ClassId> =
+            dirtyOutputClassesMap.getDirtyOutputClasses()
+                    .mapNotNull {
+                        javaSourcesProtoMap[it]?.classId
+                    }
+
+    fun isJavaClassToTrack(classId: ClassId): Boolean {
+        val jvmClassName = JvmClassName.byClassId(classId)
+        return dirtyOutputClassesMap.isDirty(jvmClassName.internalName) ||
+               javaSourcesProtoMap[jvmClassName.internalName] == null
+    }
+
    fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) {
        val dirtyClasses = dirtyOutputClassesMap
                                .getDirtyOutputClasses()
@@ -203,6 +228,7 @@ open class IncrementalJvmCache(
            constantsMap.remove(it)
            inlineFunctionsMap.remove(it)
            internalNameToSource.remove(it.internalName)
+            javaSourcesProtoMap.remove(it, changesCollector)
        }

        removeAllFromClassStorage(dirtyClasses.map { it.fqNameForClassNameWithoutDollars })
@@ -298,6 +324,29 @@ open class IncrementalJvmCache(
        }
    }

+    private inner class JavaSourcesProtoMap(storageFile: File) : BasicStringMap<SerializedJavaClass>(storageFile, JavaClassProtoMapValueExternalizer) {
+        fun process(jvmClassName: JvmClassName, newData: SerializedJavaClass, changesCollector: ChangesCollector) {
+            val key = jvmClassName.internalName
+            val oldData = storage[key]
+            storage[key] = newData
+
+            changesCollector.collectProtoChanges(oldData?.toProtoData(), newData.toProtoData())
+        }
+
+        fun remove(className: JvmClassName, changesCollector: ChangesCollector) {
+            val key = className.internalName
+            val oldValue = storage[key] ?: return
+            storage.remove(key)
+
+            changesCollector.collectProtoChanges(oldValue.toProtoData(), newData = null)
+        }
+
+        operator fun get(internalName: String) = storage[internalName]
+
+        override fun dumpValue(value: SerializedJavaClass): String =
+                java.lang.Long.toHexString(value.proto.toByteArray().md5())
+    }
+
    // todo: reuse code with InlineFunctionsMap?
    private inner class ConstantsMap(storageFile: File) : BasicStringMap<Map<String, Any>>(storageFile, ConstantsMapExternalizer) {
        private fun getConstantsMap(bytes: ByteArray): Map<String, Any> {
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import com.intellij.psi.PsiJavaFile
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.findClassAcrossModuleDependencies
+import org.jetbrains.kotlin.load.java.JavaClassesTracker
+import org.jetbrains.kotlin.load.java.descriptors.JavaClassDescriptor
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
+import org.jetbrains.kotlin.resolve.descriptorUtil.classId
+import org.jetbrains.kotlin.resolve.source.PsiSourceElement
+import org.jetbrains.kotlin.serialization.DescriptorSerializer
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.serialization.java.JavaClassProtoBuf
+import org.jetbrains.kotlin.utils.addToStdlib.safeAs
+import org.jetbrains.kotlin.utils.sure
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+
+class JavaClassesTrackerImpl(private val cache: IncrementalJvmCache) : JavaClassesTracker {
+    private val classToSourceSerialized: MutableMap<ClassId, SerializedJavaClassWithSource> = hashMapOf()
+
+    val javaClassesUpdates: Collection<SerializedJavaClassWithSource>
+        get() = classToSourceSerialized.values
+
+    private val classDescriptors: MutableList<JavaClassDescriptor> = mutableListOf()
+
+    override fun reportClass(classDescriptor: JavaClassDescriptor) {
+        val classId = classDescriptor.classId!!
+        if (!cache.isJavaClassToTrack(classId) || classDescriptor.javaSourceFile == null) return
+
+        classDescriptors.add(classDescriptor)
+    }
+
+    override fun onCompletedAnalysis(module: ModuleDescriptor) {
+        for (classId in cache.getObsoleteJavaClasses()) {
+            // Just force the loading obsolete classes
+            // We assume here that whenever an LazyJavaClassDescriptor instances is created
+            // it's being passed to JavaClassesTracker::reportClass
+            module.findClassAcrossModuleDependencies(classId)
+        }
+
+        for (classDescriptor in classDescriptors.toList()) {
+            classToSourceSerialized[classDescriptor.classId!!] =
+                    classDescriptor.convertToProto()
+        }
+    }
+}
+
+private val JavaClassDescriptor.javaSourceFile: File?
+    get() = source.safeAs<PsiSourceElement>()
+            ?.psi?.containingFile?.takeIf { it is PsiJavaFile }
+            ?.virtualFile?.path?.let(::File)
+
+fun JavaClassDescriptor.convertToProto(): SerializedJavaClassWithSource {
+    val file = javaSourceFile.sure { "convertToProto should only be called for source based classes" }
+
+    val extension = JavaClassesSerializerExtension()
+    val serializer = DescriptorSerializer.createTopLevel(extension)
+    val classProto = serializer.classProto(this).build()
+
+    val (stringTable, qualifiedNameTable) = extension.stringTable.buildProto()
+
+    return SerializedJavaClassWithSource(file, SerializedJavaClass(classProto, stringTable, qualifiedNameTable))
+}
+
+class SerializedJavaClass(
+        val proto: ProtoBuf.Class,
+        val stringTable: ProtoBuf.StringTable,
+        val qualifiedNameTable: ProtoBuf.QualifiedNameTable
+) {
+    val classId: ClassId
+        get() = NameResolverImpl(stringTable, qualifiedNameTable).getClassId(proto.fqName)
+}
+
+data class SerializedJavaClassWithSource(
+        val source: File,
+        val proto: SerializedJavaClass
+)
+
+fun SerializedJavaClass.toProtoData() = ClassProtoData(proto, NameResolverImpl(stringTable, qualifiedNameTable))
+
+val JAVA_CLASS_PROTOBUF_REGISTRY =
+        ExtensionRegistryLite.newInstance()
+                .also(JavaClassProtoBuf::registerAllExtensions)
+                // Built-ins extensions are used for annotations' serialization
+                .also(BuiltInsProtoBuf::registerAllExtensions)
+
+object JavaClassProtoMapValueExternalizer : DataExternalizer<SerializedJavaClass> {
+    override fun save(output: DataOutput, value: SerializedJavaClass) {
+        output.writeBytesWithSize(value.proto.toByteArray())
+        output.writeBytesWithSize(value.stringTable.toByteArray())
+        output.writeBytesWithSize(value.qualifiedNameTable.toByteArray())
+    }
+
+    private fun DataOutput.writeBytesWithSize(bytes: ByteArray) {
+        writeInt(bytes.size)
+        write(bytes)
+    }
+
+    private fun DataInput.readBytesWithSize(): ByteArray {
+        val bytesLength = readInt()
+        return ByteArray(bytesLength).also {
+            readFully(it, 0, bytesLength)
+        }
+    }
+
+    override fun read(input: DataInput): SerializedJavaClass {
+        val proto = ProtoBuf.Class.parseFrom(input.readBytesWithSize(), JAVA_CLASS_PROTOBUF_REGISTRY)
+        val stringTable = ProtoBuf.StringTable.parseFrom(input.readBytesWithSize(), JAVA_CLASS_PROTOBUF_REGISTRY)
+        val qualifiedNameTable = ProtoBuf.QualifiedNameTable.parseFrom(input.readBytesWithSize(), JAVA_CLASS_PROTOBUF_REGISTRY)
+
+        return SerializedJavaClass(proto, stringTable, qualifiedNameTable)
+    }
+}
@@ -79,7 +79,8 @@ fun makeCompileServices(
 fun updateIncrementalCache(
        generatedFiles: Iterable<GeneratedFile>,
        cache: IncrementalJvmCache,
-        changesCollector: ChangesCollector
+        changesCollector: ChangesCollector,
+        javaChangesTracker: JavaClassesTrackerImpl?
 ) {
    for (generatedFile in generatedFiles) {
        when {
@@ -88,6 +89,11 @@ fun updateIncrementalCache(
        }
    }

+    javaChangesTracker?.javaClassesUpdates?.forEach {
+        (source, serializedJavaClass) ->
+        cache.saveJavaClassProto(source, serializedJavaClass, changesCollector)
+    }
+
    cache.clearCacheForRemovedClasses(changesCollector)
 }

@@ -173,7 +179,7 @@ fun mapClassesFqNamesToFiles(
    for (cache in caches) {
        for (dirtyClassFqName in classesFqNames) {
            val srcFile = cache.getSourceFileIfClass(dirtyClassFqName)
-            if (srcFile == null || srcFile in excludes) continue
+            if (srcFile == null || srcFile in excludes || srcFile.isJavaFile()) continue

            reporter.report { ("Class $dirtyClassFqName caused recompilation of: ${reporter.pathsAsString(srcFile)}") }
            dirtyFiles.add(srcFile)