Support a more optimized way of storing types in metadata

Together with almost every type now there's also an id which references a type
from a separate table. Storing such ids instead of Type messages will allow to
reduce the size of the metadata for files where types are used many times over
and over again. Currently only deserialization of such types is supported,
along with the former mechanism

Original commit: 82d2e623d3

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/ProtoCompareGenerated.kt

@@ -39,11 +39,17 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsPackageProperty(old, new)) return false
 
+        if (old.hasTypeTable() != new.hasTypeTable()) return false
+        if (old.hasTypeTable()) {
+            if (!checkEquals(old.typeTable, new.typeTable)) return false
+        }
+
         return true
     }
     public enum class ProtoBufPackageKind {
         FUNCTION_LIST,
-        PROPERTY_LIST
+        PROPERTY_LIST,
+        TYPE_TABLE
     }
 
     public fun difference(old: ProtoBuf.Package, new: ProtoBuf.Package): EnumSet<ProtoBufPackageKind> {
@@ -53,6 +59,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsPackageProperty(old, new)) result.add(ProtoBufPackageKind.PROPERTY_LIST)
 
+        if (old.hasTypeTable() != new.hasTypeTable()) result.add(ProtoBufPackageKind.TYPE_TABLE)
+        if (old.hasTypeTable()) {
+            if (!checkEquals(old.typeTable, new.typeTable)) result.add(ProtoBufPackageKind.TYPE_TABLE)
+        }
+
         return result
     }
 
@@ -73,6 +84,8 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsClassSupertype(old, new)) return false
 
+        if (!checkEqualsClassSupertypeId(old, new)) return false
+
         if (!checkEqualsClassNestedClassName(old, new)) return false
 
         if (!checkEqualsClassConstructor(old, new)) return false
@@ -83,6 +96,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsClassEnumEntry(old, new)) return false
 
+        if (old.hasTypeTable() != new.hasTypeTable()) return false
+        if (old.hasTypeTable()) {
+            if (!checkEquals(old.typeTable, new.typeTable)) return false
+        }
+
         if (old.getExtensionCount(JvmProtoBuf.classAnnotation) != new.getExtensionCount(JvmProtoBuf.classAnnotation)) return false
 
         for(i in 0..old.getExtensionCount(JvmProtoBuf.classAnnotation) - 1) {
@@ -97,11 +115,13 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
         COMPANION_OBJECT_NAME,
         TYPE_PARAMETER_LIST,
         SUPERTYPE_LIST,
+        SUPERTYPE_ID_LIST,
         NESTED_CLASS_NAME_LIST,
         CONSTRUCTOR_LIST,
         FUNCTION_LIST,
         PROPERTY_LIST,
         ENUM_ENTRY_LIST,
+        TYPE_TABLE,
         CLASS_ANNOTATION_LIST
     }
 
@@ -124,6 +144,8 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsClassSupertype(old, new)) result.add(ProtoBufClassKind.SUPERTYPE_LIST)
 
+        if (!checkEqualsClassSupertypeId(old, new)) result.add(ProtoBufClassKind.SUPERTYPE_ID_LIST)
+
         if (!checkEqualsClassNestedClassName(old, new)) result.add(ProtoBufClassKind.NESTED_CLASS_NAME_LIST)
 
         if (!checkEqualsClassConstructor(old, new)) result.add(ProtoBufClassKind.CONSTRUCTOR_LIST)
@@ -134,6 +156,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsClassEnumEntry(old, new)) result.add(ProtoBufClassKind.ENUM_ENTRY_LIST)
 
+        if (old.hasTypeTable() != new.hasTypeTable()) result.add(ProtoBufClassKind.TYPE_TABLE)
+        if (old.hasTypeTable()) {
+            if (!checkEquals(old.typeTable, new.typeTable)) result.add(ProtoBufClassKind.TYPE_TABLE)
+        }
+
         if (old.getExtensionCount(JvmProtoBuf.classAnnotation) != new.getExtensionCount(JvmProtoBuf.classAnnotation)) result.add(ProtoBufClassKind.CLASS_ANNOTATION_LIST)
 
         for(i in 0..old.getExtensionCount(JvmProtoBuf.classAnnotation) - 1) {
@@ -151,7 +178,15 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkStringEquals(old.name, new.name)) return false
 
-        if (!checkEquals(old.returnType, new.returnType)) return false
+        if (old.hasReturnType() != new.hasReturnType()) return false
+        if (old.hasReturnType()) {
+            if (!checkEquals(old.returnType, new.returnType)) return false
+        }
+
+        if (old.hasReturnTypeId() != new.hasReturnTypeId()) return false
+        if (old.hasReturnTypeId()) {
+            if (old.returnTypeId != new.returnTypeId) return false
+        }
 
         if (!checkEqualsFunctionTypeParameter(old, new)) return false
 
@@ -160,8 +195,18 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
             if (!checkEquals(old.receiverType, new.receiverType)) return false
         }
 
+        if (old.hasReceiverTypeId() != new.hasReceiverTypeId()) return false
+        if (old.hasReceiverTypeId()) {
+            if (old.receiverTypeId != new.receiverTypeId) return false
+        }
+
         if (!checkEqualsFunctionValueParameter(old, new)) return false
 
+        if (old.hasTypeTable() != new.hasTypeTable()) return false
+        if (old.hasTypeTable()) {
+            if (!checkEquals(old.typeTable, new.typeTable)) return false
+        }
+
         if (old.hasExtension(JvmProtoBuf.methodSignature) != new.hasExtension(JvmProtoBuf.methodSignature)) return false
         if (old.hasExtension(JvmProtoBuf.methodSignature)) {
             if (!checkEquals(old.getExtension(JvmProtoBuf.methodSignature), new.getExtension(JvmProtoBuf.methodSignature))) return false
@@ -183,7 +228,15 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkStringEquals(old.name, new.name)) return false
 
-        if (!checkEquals(old.returnType, new.returnType)) return false
+        if (old.hasReturnType() != new.hasReturnType()) return false
+        if (old.hasReturnType()) {
+            if (!checkEquals(old.returnType, new.returnType)) return false
+        }
+
+        if (old.hasReturnTypeId() != new.hasReturnTypeId()) return false
+        if (old.hasReturnTypeId()) {
+            if (old.returnTypeId != new.returnTypeId) return false
+        }
 
         if (!checkEqualsPropertyTypeParameter(old, new)) return false
 
@@ -192,6 +245,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
             if (!checkEquals(old.receiverType, new.receiverType)) return false
         }
 
+        if (old.hasReceiverTypeId() != new.hasReceiverTypeId()) return false
+        if (old.hasReceiverTypeId()) {
+            if (old.receiverTypeId != new.receiverTypeId) return false
+        }
+
         if (old.hasSetterValueParameter() != new.hasSetterValueParameter()) return false
         if (old.hasSetterValueParameter()) {
             if (!checkEquals(old.setterValueParameter, new.setterValueParameter)) return false
@@ -220,6 +278,17 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
         return true
     }
 
+    open fun checkEquals(old: ProtoBuf.TypeTable, new: ProtoBuf.TypeTable): Boolean {
+        if (!checkEqualsTypeTableType(old, new)) return false
+
+        if (old.hasFirstNullable() != new.hasFirstNullable()) return false
+        if (old.hasFirstNullable()) {
+            if (old.firstNullable != new.firstNullable) return false
+        }
+
+        return true
+    }
+
     open fun checkEquals(old: ProtoBuf.TypeParameter, new: ProtoBuf.TypeParameter): Boolean {
         if (old.id != new.id) return false
 
@@ -237,6 +306,8 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkEqualsTypeParameterUpperBound(old, new)) return false
 
+        if (!checkEqualsTypeParameterUpperBoundId(old, new)) return false
+
         return true
     }
 
@@ -258,6 +329,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
             if (!checkEquals(old.flexibleUpperBound, new.flexibleUpperBound)) return false
         }
 
+        if (old.hasFlexibleUpperBoundId() != new.hasFlexibleUpperBoundId()) return false
+        if (old.hasFlexibleUpperBoundId()) {
+            if (old.flexibleUpperBoundId != new.flexibleUpperBoundId) return false
+        }
+
         if (old.hasClassName() != new.hasClassName()) return false
         if (old.hasClassName()) {
             if (!checkClassIdEquals(old.className, new.className)) return false
@@ -314,13 +390,26 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
 
         if (!checkStringEquals(old.name, new.name)) return false
 
-        if (!checkEquals(old.type, new.type)) return false
+        if (old.hasType() != new.hasType()) return false
+        if (old.hasType()) {
+            if (!checkEquals(old.type, new.type)) return false
+        }
+
+        if (old.hasTypeId() != new.hasTypeId()) return false
+        if (old.hasTypeId()) {
+            if (old.typeId != new.typeId) return false
+        }
 
         if (old.hasVarargElementType() != new.hasVarargElementType()) return false
         if (old.hasVarargElementType()) {
             if (!checkEquals(old.varargElementType, new.varargElementType)) return false
         }
 
+        if (old.hasVarargElementTypeId() != new.hasVarargElementTypeId()) return false
+        if (old.hasVarargElementTypeId()) {
+            if (old.varargElementTypeId != new.varargElementTypeId) return false
+        }
+
         if (old.hasExtension(JvmProtoBuf.index) != new.hasExtension(JvmProtoBuf.index)) return false
         if (old.hasExtension(JvmProtoBuf.index)) {
             if (old.getExtension(JvmProtoBuf.index) != new.getExtension(JvmProtoBuf.index)) return false
@@ -378,6 +467,11 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
             if (!checkEquals(old.type, new.type)) return false
         }
 
+        if (old.hasTypeId() != new.hasTypeId()) return false
+        if (old.hasTypeId()) {
+            if (old.typeId != new.typeId) return false
+        }
+
         return true
     }
 
@@ -494,6 +588,16 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
         return true
     }
 
+    open fun checkEqualsClassSupertypeId(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean {
+        if (old.supertypeIdCount != new.supertypeIdCount) return false
+
+        for(i in 0..old.supertypeIdCount - 1) {
+            if (old.getSupertypeId(i) != new.getSupertypeId(i)) return false
+        }
+
+        return true
+    }
+
     open fun checkEqualsClassNestedClassName(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean {
         if (old.nestedClassNameCount != new.nestedClassNameCount) return false
 
@@ -574,6 +678,16 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
         return true
     }
 
+    open fun checkEqualsTypeTableType(old: ProtoBuf.TypeTable, new: ProtoBuf.TypeTable): Boolean {
+        if (old.typeCount != new.typeCount) return false
+
+        for(i in 0..old.typeCount - 1) {
+            if (!checkEquals(old.getType(i), new.getType(i))) return false
+        }
+
+        return true
+    }
+
     open fun checkEqualsTypeParameterUpperBound(old: ProtoBuf.TypeParameter, new: ProtoBuf.TypeParameter): Boolean {
         if (old.upperBoundCount != new.upperBoundCount) return false
 
@@ -584,6 +698,16 @@ open class ProtoCompareGenerated(public val oldNameResolver: NameResolver, publi
         return true
     }
 
+    open fun checkEqualsTypeParameterUpperBoundId(old: ProtoBuf.TypeParameter, new: ProtoBuf.TypeParameter): Boolean {
+        if (old.upperBoundIdCount != new.upperBoundIdCount) return false
+
+        for(i in 0..old.upperBoundIdCount - 1) {
+            if (old.getUpperBoundId(i) != new.getUpperBoundId(i)) return false
+        }
+
+        return true
+    }
+
     open fun checkEqualsTypeArgument(old: ProtoBuf.Type, new: ProtoBuf.Type): Boolean {
         if (old.argumentCount != new.argumentCount) return false
 
@@ -666,6 +790,10 @@ public fun ProtoBuf.Package.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes:
         hashCode = 31 * hashCode + getProperty(i).hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasTypeTable()) {
+        hashCode = 31 * hashCode + typeTable.hashCode(stringIndexes, fqNameIndexes)
+    }
+
     return hashCode
 }
 
@@ -690,6 +818,10 @@ public fun ProtoBuf.Class.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (
         hashCode = 31 * hashCode + getSupertype(i).hashCode(stringIndexes, fqNameIndexes)
     }
 
+    for(i in 0..supertypeIdCount - 1) {
+        hashCode = 31 * hashCode + getSupertypeId(i)
+    }
+
     for(i in 0..nestedClassNameCount - 1) {
         hashCode = 31 * hashCode + stringIndexes(getNestedClassName(i))
     }
@@ -710,6 +842,10 @@ public fun ProtoBuf.Class.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (
         hashCode = 31 * hashCode + stringIndexes(getEnumEntry(i))
     }
 
+    if (hasTypeTable()) {
+        hashCode = 31 * hashCode + typeTable.hashCode(stringIndexes, fqNameIndexes)
+    }
+
     for(i in 0..getExtensionCount(JvmProtoBuf.classAnnotation) - 1) {
         hashCode = 31 * hashCode + getExtension(JvmProtoBuf.classAnnotation, i).hashCode(stringIndexes, fqNameIndexes)
     }
@@ -726,7 +862,13 @@ public fun ProtoBuf.Function.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes
 
     hashCode = 31 * hashCode + stringIndexes(name)
 
-    hashCode = 31 * hashCode + returnType.hashCode(stringIndexes, fqNameIndexes)
+    if (hasReturnType()) {
+        hashCode = 31 * hashCode + returnType.hashCode(stringIndexes, fqNameIndexes)
+    }
+
+    if (hasReturnTypeId()) {
+        hashCode = 31 * hashCode + returnTypeId
+    }
 
     for(i in 0..typeParameterCount - 1) {
         hashCode = 31 * hashCode + getTypeParameter(i).hashCode(stringIndexes, fqNameIndexes)
@@ -736,10 +878,18 @@ public fun ProtoBuf.Function.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes
         hashCode = 31 * hashCode + receiverType.hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasReceiverTypeId()) {
+        hashCode = 31 * hashCode + receiverTypeId
+    }
+
     for(i in 0..valueParameterCount - 1) {
         hashCode = 31 * hashCode + getValueParameter(i).hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasTypeTable()) {
+        hashCode = 31 * hashCode + typeTable.hashCode(stringIndexes, fqNameIndexes)
+    }
+
     if (hasExtension(JvmProtoBuf.methodSignature)) {
         hashCode = 31 * hashCode + getExtension(JvmProtoBuf.methodSignature).hashCode(stringIndexes, fqNameIndexes)
     }
@@ -760,7 +910,13 @@ public fun ProtoBuf.Property.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes
 
     hashCode = 31 * hashCode + stringIndexes(name)
 
-    hashCode = 31 * hashCode + returnType.hashCode(stringIndexes, fqNameIndexes)
+    if (hasReturnType()) {
+        hashCode = 31 * hashCode + returnType.hashCode(stringIndexes, fqNameIndexes)
+    }
+
+    if (hasReturnTypeId()) {
+        hashCode = 31 * hashCode + returnTypeId
+    }
 
     for(i in 0..typeParameterCount - 1) {
         hashCode = 31 * hashCode + getTypeParameter(i).hashCode(stringIndexes, fqNameIndexes)
@@ -770,6 +926,10 @@ public fun ProtoBuf.Property.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes
         hashCode = 31 * hashCode + receiverType.hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasReceiverTypeId()) {
+        hashCode = 31 * hashCode + receiverTypeId
+    }
+
     if (hasSetterValueParameter()) {
         hashCode = 31 * hashCode + setterValueParameter.hashCode(stringIndexes, fqNameIndexes)
     }
@@ -793,6 +953,20 @@ public fun ProtoBuf.Property.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes
     return hashCode
 }
 
+public fun ProtoBuf.TypeTable.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (Int) -> Int): Int {
+    var hashCode = 1
+
+    for(i in 0..typeCount - 1) {
+        hashCode = 31 * hashCode + getType(i).hashCode(stringIndexes, fqNameIndexes)
+    }
+
+    if (hasFirstNullable()) {
+        hashCode = 31 * hashCode + firstNullable
+    }
+
+    return hashCode
+}
+
 public fun ProtoBuf.TypeParameter.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (Int) -> Int): Int {
     var hashCode = 1
 
@@ -812,6 +986,10 @@ public fun ProtoBuf.TypeParameter.hashCode(stringIndexes: (Int) -> Int, fqNameIn
         hashCode = 31 * hashCode + getUpperBound(i).hashCode(stringIndexes, fqNameIndexes)
     }
 
+    for(i in 0..upperBoundIdCount - 1) {
+        hashCode = 31 * hashCode + getUpperBoundId(i)
+    }
+
     return hashCode
 }
 
@@ -834,6 +1012,10 @@ public fun ProtoBuf.Type.hashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (I
         hashCode = 31 * hashCode + flexibleUpperBound.hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasFlexibleUpperBoundId()) {
+        hashCode = 31 * hashCode + flexibleUpperBoundId
+    }
+
     if (hasClassName()) {
         hashCode = 31 * hashCode + fqNameIndexes(className)
     }
@@ -892,12 +1074,22 @@ public fun ProtoBuf.ValueParameter.hashCode(stringIndexes: (Int) -> Int, fqNameI
 
     hashCode = 31 * hashCode + stringIndexes(name)
 
-    hashCode = 31 * hashCode + type.hashCode(stringIndexes, fqNameIndexes)
+    if (hasType()) {
+        hashCode = 31 * hashCode + type.hashCode(stringIndexes, fqNameIndexes)
+    }
+
+    if (hasTypeId()) {
+        hashCode = 31 * hashCode + typeId
+    }
 
     if (hasVarargElementType()) {
         hashCode = 31 * hashCode + varargElementType.hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasVarargElementTypeId()) {
+        hashCode = 31 * hashCode + varargElementTypeId
+    }
+
     if (hasExtension(JvmProtoBuf.index)) {
         hashCode = 31 * hashCode + getExtension(JvmProtoBuf.index)
     }
@@ -952,6 +1144,10 @@ public fun ProtoBuf.Type.Argument.hashCode(stringIndexes: (Int) -> Int, fqNameIn
         hashCode = 31 * hashCode + type.hashCode(stringIndexes, fqNameIndexes)
     }
 
+    if (hasTypeId()) {
+        hashCode = 31 * hashCode + typeId
+    }
+
     return hashCode
 }
 

jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt

@@ -210,6 +210,9 @@ private class DifferenceCalculatorForClass(oldData: ProtoMapValue, newData: Prot
                     names.addAll(calcDifferenceForNonPrivateMembers(ProtoBuf.Class::getPropertyList))
                 ProtoBufClassKind.ENUM_ENTRY_LIST ->
                     names.addAll(calcDifferenceForNames(oldProto.enumEntryList, newProto.enumEntryList))
+                ProtoBufClassKind.TYPE_TABLE -> {
+                    // TODO
+                }
                 ProtoBufClassKind.FLAGS,
                 ProtoBufClassKind.FQ_NAME,
                 ProtoBufClassKind.TYPE_PARAMETER_LIST,
@@ -257,6 +260,9 @@ private class DifferenceCalculatorForPackageFacade(oldData: ProtoMapValue, newDa
                     names.addAll(calcDifferenceForNonPrivateMembers(ProtoBuf.Package::getFunctionList))
                 ProtoBufPackageKind.PROPERTY_LIST ->
                     names.addAll(calcDifferenceForNonPrivateMembers(ProtoBuf.Package::getPropertyList))
+                ProtoBufPackageKind.TYPE_TABLE -> {
+                    // TODO
+                }
                 else ->
                     throw IllegalArgumentException("Unsupported kind: $kind")
             }