Write java signature of serialized descriptors efficiently

Instead of storing a string with the signature, write the name, return type and
parameter types separately, reusing the name table. Refactor the name table to
allow it to store not only names of Kotlin entities, but also arbitrary names
and fq-names
This commit is contained in:
Alexander Udalov
2013-07-11 17:12:01 +04:00
parent eee9bb5a12
commit 453b0e8f10
13 changed files with 1841 additions and 76 deletions
@@ -17,14 +17,15 @@
package org.jetbrains.jet.codegen;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.codegen.signature.JvmMethodSignature;
import org.jetbrains.asm4.commons.Method;
import org.jetbrains.jet.codegen.state.JetTypeMapper;
import org.jetbrains.jet.descriptors.serialization.JavaProtoBufUtil;
import org.jetbrains.jet.descriptors.serialization.NameTable;
import org.jetbrains.jet.descriptors.serialization.ProtoBuf;
import org.jetbrains.jet.descriptors.serialization.SerializerExtension;
import org.jetbrains.jet.lang.descriptors.CallableMemberDescriptor;
import org.jetbrains.jet.lang.descriptors.FunctionDescriptor;
import org.jetbrains.jet.lang.descriptors.PropertyDescriptor;
public class JavaSerializerExtension extends SerializerExtension {
private final JetTypeMapper typeMapper;
@@ -33,20 +34,20 @@ public class JavaSerializerExtension extends SerializerExtension {
this.typeMapper = typeMapper;
}
// TODO: mapSignature should be done upon generation of the member instead, because we don't know enough at this point to map correctly
@Override
public void serializeCallable(@NotNull CallableMemberDescriptor callable, @NotNull ProtoBuf.Callable.Builder proto) {
JvmMethodSignature signature = mapSignature(callable);
if (signature != null) {
JavaProtoBufUtil.saveJavaSignature(proto, signature.getAsmMethod().toString());
public void serializeCallable(
@NotNull CallableMemberDescriptor callable,
@NotNull ProtoBuf.Callable.Builder proto,
@NotNull NameTable nameTable
) {
if (callable instanceof FunctionDescriptor) {
Method method = typeMapper.mapSignature((FunctionDescriptor) callable).getAsmMethod();
JavaProtoBufUtil.saveMethodSignature(proto, method, nameTable);
}
}
// TODO: this should be done upon generation of the member instead, because we don't know enough at this point to map correctly
@Nullable
private JvmMethodSignature mapSignature(@NotNull CallableMemberDescriptor descriptor) {
if (descriptor instanceof FunctionDescriptor) {
return typeMapper.mapSignature((FunctionDescriptor) descriptor);
else if (callable instanceof PropertyDescriptor) {
PropertyDescriptor property = (PropertyDescriptor) callable;
// TODO
}
return null;
}
}
@@ -10,6 +10,8 @@
<orderEntry type="library" name="protobuf-java" level="project" />
<orderEntry type="module" module-name="serialization" />
<orderEntry type="library" name="annotations" level="project" />
<orderEntry type="library" name="asm" level="project" />
<orderEntry type="module" module-name="frontend" />
</component>
</module>
@@ -21,6 +21,42 @@ import "compiler/frontend/serialization/src/descriptors.proto";
option java_outer_classname = "JavaProtoBuf";
option optimize_for = LITE_RUNTIME;
extend Callable {
optional string java_signature = 100;
message JavaType {
enum PrimitiveType {
// These values correspond to ASM Type sorts
VOID = 0;
BOOLEAN = 1;
CHAR = 2;
BYTE = 3;
SHORT = 4;
INT = 5;
FLOAT = 6;
LONG = 7;
DOUBLE = 8;
}
// One of these should be present
optional PrimitiveType primitive_type = 1;
optional int32 class_fq_name = 2;
optional int32 array_dimension = 3 [default = 0];
}
message JavaMethodSignature {
required int32 name = 1;
required JavaType return_type = 2;
repeated JavaType parameter_type = 3;
}
message JavaPropertySignature {
required JavaType type = 1;
optional int32 field_name = 2;
optional JavaMethodSignature getter = 3;
optional JavaMethodSignature setter = 4;
}
extend Callable {
optional JavaMethodSignature method_signature = 100;
optional JavaPropertySignature property_signature = 101;
}
@@ -19,22 +19,138 @@ package org.jetbrains.jet.descriptors.serialization;
import com.google.protobuf.ExtensionRegistryLite;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.asm4.Type;
import org.jetbrains.asm4.commons.Method;
import org.jetbrains.jet.lang.resolve.name.FqName;
import org.jetbrains.jet.lang.resolve.name.Name;
import static org.jetbrains.jet.descriptors.serialization.JavaProtoBuf.javaSignature;
import java.util.Arrays;
import static org.jetbrains.asm4.Type.*;
public class JavaProtoBufUtil {
private JavaProtoBufUtil() {
}
@Nullable
public static String loadJavaSignature(@NotNull ProtoBuf.Callable callable) {
return callable.hasExtension(javaSignature) ? callable.getExtension(javaSignature) : null;
public static String loadMethodSignature(@NotNull ProtoBuf.Callable proto, @NotNull NameResolver nameResolver) {
if (!proto.hasExtension(JavaProtoBuf.methodSignature)) return null;
JavaProtoBuf.JavaMethodSignature signature = proto.getExtension(JavaProtoBuf.methodSignature);
return new Deserializer(nameResolver).methodSignature(signature).toString();
}
public static void saveJavaSignature(@NotNull ProtoBuf.Callable.Builder callable, @NotNull String signature) {
callable.setExtension(javaSignature, signature);
public static void saveMethodSignature(
@NotNull ProtoBuf.Callable.Builder proto,
@NotNull Method method,
@NotNull NameTable nameTable
) {
proto.setExtension(JavaProtoBuf.methodSignature, new Serializer(nameTable).methodSignature(method));
}
private static class Serializer {
private final NameTable nameTable;
public Serializer(@NotNull NameTable nameTable) {
this.nameTable = nameTable;
}
@NotNull
public JavaProtoBuf.JavaMethodSignature methodSignature(@NotNull Method method) {
JavaProtoBuf.JavaMethodSignature.Builder signature = JavaProtoBuf.JavaMethodSignature.newBuilder();
signature.setName(nameTable.getSimpleNameIndex(Name.guess(method.getName())));
signature.setReturnType(type(method.getReturnType()));
for (Type type : method.getArgumentTypes()) {
signature.addParameterType(type(type));
}
return signature.build();
}
@NotNull
private JavaProtoBuf.JavaType type(@NotNull Type givenType) {
JavaProtoBuf.JavaType.Builder builder = JavaProtoBuf.JavaType.newBuilder();
int arrayDimension = 0;
Type type = givenType;
while (type.getSort() == Type.ARRAY) {
arrayDimension++;
type = type.getElementType();
}
if (arrayDimension != 0) {
builder.setArrayDimension(arrayDimension);
}
if (type.getSort() == Type.OBJECT) {
FqName fqName = internalNameToFqName(type.getInternalName());
builder.setClassFqName(nameTable.getFqNameIndex(fqName));
}
else {
builder.setPrimitiveType(JavaProtoBuf.JavaType.PrimitiveType.valueOf(type.getSort()));
}
return builder.build();
}
@NotNull
private static FqName internalNameToFqName(@NotNull String internalName) {
return FqName.fromSegments(Arrays.asList(internalName.split("/")));
}
}
private static class Deserializer {
// These types are ordered according to their sorts, this is significant for deserialization
private static final Type[] PRIMITIVE_TYPES = new Type[]
{ VOID_TYPE, BOOLEAN_TYPE, CHAR_TYPE, BYTE_TYPE, SHORT_TYPE, INT_TYPE, FLOAT_TYPE, LONG_TYPE, DOUBLE_TYPE };
private final NameResolver nameResolver;
public Deserializer(@NotNull NameResolver nameResolver) {
this.nameResolver = nameResolver;
}
@NotNull
public Method methodSignature(@NotNull JavaProtoBuf.JavaMethodSignature signature) {
String name = nameResolver.getName(signature.getName()).asString();
Type returnType = type(signature.getReturnType());
int parameters = signature.getParameterTypeCount();
Type[] parameterTypes = new Type[parameters];
for (int i = 0; i < parameters; i++) {
parameterTypes[i] = type(signature.getParameterType(i));
}
return new Method(name, returnType, parameterTypes);
}
@NotNull
private Type type(@NotNull JavaProtoBuf.JavaType type) {
Type result;
if (type.hasPrimitiveType()) {
result = PRIMITIVE_TYPES[type.getPrimitiveType().ordinal()];
}
else {
result = Type.getObjectType(fqNameToInternalName(nameResolver.getFqName(type.getClassFqName())));
}
StringBuilder brackets = new StringBuilder(type.getArrayDimension());
for (int i = 0; i < type.getArrayDimension(); i++) {
brackets.append('[');
}
return Type.getType(brackets + result.getDescriptor());
}
@NotNull
private static String fqNameToInternalName(@NotNull FqName fqName) {
return fqName.asString().replace('.', '/');
}
}
@NotNull
public static ExtensionRegistryLite getExtensionRegistry() {
ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance();
@@ -22,6 +22,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.asm4.*;
import org.jetbrains.jet.descriptors.serialization.JavaProtoBufUtil;
import org.jetbrains.jet.descriptors.serialization.NameResolver;
import org.jetbrains.jet.descriptors.serialization.ProtoBuf;
import org.jetbrains.jet.descriptors.serialization.descriptors.AnnotationDeserializer;
import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
@@ -175,17 +176,19 @@ public class AnnotationDescriptorDeserializer implements AnnotationDeserializer
@Override
public List<AnnotationDescriptor> loadCallableAnnotations(
@NotNull ClassOrNamespaceDescriptor container,
@NotNull ProtoBuf.Callable callableProto
@NotNull ProtoBuf.Callable proto,
@NotNull NameResolver nameResolver,
@NotNull AnnotatedCallableKind kind
) {
String signature = getCallableSignature(proto, nameResolver, kind);
if (signature == null) return Collections.emptyList();
VirtualFile file = findVirtualFileByDescriptor(container);
try {
// TODO: calculate this only once for each container
Map<String, List<AnnotationDescriptor>> memberAnnotations = loadMemberAnnotationsFromFile(file);
String signature = JavaProtoBufUtil.loadJavaSignature(callableProto);
if (signature == null) return Collections.emptyList();
List<AnnotationDescriptor> annotations = memberAnnotations.get(signature);
return annotations == null ? Collections.<AnnotationDescriptor>emptyList() : annotations;
}
@@ -194,6 +197,21 @@ public class AnnotationDescriptorDeserializer implements AnnotationDeserializer
}
}
@Nullable
private static String getCallableSignature(
@NotNull ProtoBuf.Callable proto,
@NotNull NameResolver nameResolver,
@NotNull AnnotatedCallableKind kind
) {
switch (kind) {
case FUNCTION:
return JavaProtoBufUtil.loadMethodSignature(proto, nameResolver);
// TODO: getters, setters
default:
return null;
}
}
@NotNull
private Map<String, List<AnnotationDescriptor>> loadMemberAnnotationsFromFile(@NotNull VirtualFile file) throws IOException {
final Map<String, List<AnnotationDescriptor>> memberAnnotations = new HashMap<String, List<AnnotationDescriptor>>();
@@ -37,6 +37,7 @@ import java.util.List;
import static org.jetbrains.jet.descriptors.serialization.ProtoBuf.Callable;
import static org.jetbrains.jet.descriptors.serialization.ProtoBuf.TypeParameter;
import static org.jetbrains.jet.descriptors.serialization.TypeDeserializer.TypeParameterResolver.NONE;
import static org.jetbrains.jet.descriptors.serialization.descriptors.AnnotationDeserializer.AnnotatedCallableKind;
public class DescriptorDeserializer {
@@ -153,7 +154,7 @@ public class DescriptorDeserializer {
boolean isNotDefault = proto.hasGetterFlags() && Flags.IS_NOT_DEFAULT.get(getterFlags);
if (isNotDefault) {
getter = new PropertyGetterDescriptorImpl(
property, Collections.<AnnotationDescriptor>emptyList(),
property, getAnnotations(proto, getterFlags, AnnotatedCallableKind.PROPERTY_GETTER),
modality(Flags.MODALITY.get(getterFlags)), visibility(Flags.VISIBILITY.get(getterFlags)),
isNotDefault, !isNotDefault, property.getKind()
);
@@ -169,7 +170,7 @@ public class DescriptorDeserializer {
boolean isNotDefault = proto.hasSetterFlags() && Flags.IS_NOT_DEFAULT.get(setterFlags);
if (isNotDefault) {
setter = new PropertySetterDescriptorImpl(
property, getAnnotations(proto, setterFlags),
property, getAnnotations(proto, setterFlags, AnnotatedCallableKind.PROPERTY_SETTER),
modality(Flags.MODALITY.get(setterFlags)), visibility(Flags.VISIBILITY.get(setterFlags)),
isNotDefault, !isNotDefault, property.getKind()
);
@@ -192,7 +193,7 @@ public class DescriptorDeserializer {
private PropertyDescriptorImpl createPropertyDescriptor(@NotNull Callable proto) {
int flags = proto.getFlags();
Name name = nameResolver.getName(proto.getName());
List<AnnotationDescriptor> annotations = getAnnotations(proto, proto.getFlags());
List<AnnotationDescriptor> annotations = Collections.emptyList(); // TODO: getAnnotations(proto, flags, PROPERTY);
Visibility visibility = visibility(Flags.VISIBILITY.get(flags));
Callable.CallableKind callableKind = Flags.CALLABLE_KIND.get(flags);
@@ -221,7 +222,7 @@ public class DescriptorDeserializer {
int flags = proto.getFlags();
SimpleFunctionDescriptorImpl function = new SimpleFunctionDescriptorImpl(
containingDeclaration,
getAnnotations(proto, proto.getFlags()),
getAnnotations(proto, proto.getFlags(), AnnotatedCallableKind.FUNCTION),
nameResolver.getName(proto.getName()),
memberKind(Flags.MEMBER_KIND.get(flags))
);
@@ -252,7 +253,7 @@ public class DescriptorDeserializer {
ClassDescriptor classDescriptor = (ClassDescriptor) containingDeclaration;
ConstructorDescriptorImpl descriptor = new ConstructorDescriptorImpl(
classDescriptor,
getAnnotations(proto, proto.getFlags()),
getAnnotations(proto, proto.getFlags(), AnnotatedCallableKind.FUNCTION),
// TODO: primary
true);
List<TypeParameterDescriptor> typeParameters = new ArrayList<TypeParameterDescriptor>(proto.getTypeParameterCount());
@@ -268,11 +269,12 @@ public class DescriptorDeserializer {
}
@NotNull
private List<AnnotationDescriptor> getAnnotations(@NotNull Callable proto, int flags) {
private List<AnnotationDescriptor> getAnnotations(@NotNull Callable proto, int flags, @NotNull AnnotatedCallableKind kind) {
assert containingDeclaration instanceof ClassOrNamespaceDescriptor
: "Only members in classes or namespaces should be serialized: " + containingDeclaration;
return Flags.HAS_ANNOTATIONS.get(flags)
? annotationDeserializer.loadCallableAnnotations((ClassOrNamespaceDescriptor) containingDeclaration, proto)
? annotationDeserializer
.loadCallableAnnotations((ClassOrNamespaceDescriptor) containingDeclaration, proto, nameResolver, kind)
: Collections.<AnnotationDescriptor>emptyList();
}
@@ -212,7 +212,7 @@ public class DescriptorSerializer {
builder.setReturnType(local.type(getSerializableReturnType(descriptor.getReturnType())));
extension.serializeCallable(descriptor, builder);
extension.serializeCallable(descriptor, builder, nameTable);
return builder;
}
@@ -76,11 +76,7 @@ public class NameResolver {
}
@Nullable
private QualifiedName renderFqName(
StringBuilder sb,
QualifiedName fqNameProto,
QualifiedName.Kind kind
) {
private QualifiedName renderFqName(StringBuilder sb, QualifiedName fqNameProto, QualifiedName.Kind kind) {
QualifiedName result = null;
if (fqNameProto.hasParentQualifiedName()) {
QualifiedName parentProto = qualifiedNames.getQualifiedName(fqNameProto.getParentQualifiedName());
@@ -95,4 +91,14 @@ public class NameResolver {
sb.append(simpleNames.getName(fqNameProto.getShortName()));
return result;
}
@NotNull
public FqName getFqName(int index) {
QualifiedName qualifiedName = qualifiedNames.getQualifiedName(index);
Name shortName = getName(qualifiedName.getShortName());
if (!qualifiedName.hasParentQualifiedName()) {
return FqName.topLevel(shortName);
}
return getFqName(qualifiedName.getParentQualifiedName()).child(shortName);
}
}
@@ -19,10 +19,11 @@ package org.jetbrains.jet.descriptors.serialization;
import gnu.trove.TObjectHashingStrategy;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
import org.jetbrains.jet.lang.descriptors.ClassOrNamespaceDescriptor;
import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor;
import org.jetbrains.jet.lang.descriptors.NamespaceDescriptor;
import org.jetbrains.jet.lang.descriptors.impl.NamespaceDescriptorParent;
import org.jetbrains.jet.lang.resolve.DescriptorUtils;
import org.jetbrains.jet.lang.resolve.name.FqName;
import org.jetbrains.jet.lang.resolve.name.Name;
import java.util.List;
@@ -42,9 +43,7 @@ public class NameTable {
}
@Override
public boolean equals(
QualifiedName.Builder o1, QualifiedName.Builder o2
) {
public boolean equals(QualifiedName.Builder o1, QualifiedName.Builder o2) {
return o1.getParentQualifiedName() == o2.getParentQualifiedName()
&& o1.getShortName() == o2.getShortName()
&& o1.getKind() == o2.getKind();
@@ -71,18 +70,17 @@ public class NameTable {
return simpleNames.intern(name.asString());
}
public int getFqNameIndex(@NotNull ClassDescriptor classDescriptor) {
public int getFqNameIndex(@NotNull ClassOrNamespaceDescriptor descriptor) {
QualifiedName.Builder builder = QualifiedName.newBuilder();
builder.setKind(QualifiedName.Kind.CLASS);
builder.setShortName(getSimpleNameIndex(classDescriptor.getName()));
if (descriptor instanceof ClassDescriptor) {
builder.setKind(QualifiedName.Kind.CLASS);
}
builder.setShortName(getSimpleNameIndex(descriptor.getName()));
DeclarationDescriptor containingDeclaration = classDescriptor.getContainingDeclaration();
DeclarationDescriptor containingDeclaration = descriptor.getContainingDeclaration();
if (containingDeclaration instanceof NamespaceDescriptor) {
NamespaceDescriptor namespaceDescriptor = (NamespaceDescriptor) containingDeclaration;
if (DescriptorUtils.isRootNamespace(namespaceDescriptor)) {
builder.clearParentQualifiedName();
}
else {
if (!DescriptorUtils.isRootNamespace(namespaceDescriptor)) {
builder.setParentQualifiedName(getFqNameIndex(namespaceDescriptor));
}
}
@@ -91,28 +89,22 @@ public class NameTable {
builder.setParentQualifiedName(getFqNameIndex(outerClass));
}
else {
throw new IllegalStateException("FQ names are only stored for top-level or inner classes: " + classDescriptor);
throw new IllegalStateException("FQ names are only stored for top-level or inner classes: " + descriptor);
}
return qualifiedNames.intern(builder);
}
public int getFqNameIndex(@NotNull NamespaceDescriptor namespaceDescriptor) {
QualifiedName.Builder builder = QualifiedName.newBuilder();
//default: builder.setKind(QualifiedNameTable.QualifiedName.Kind.PACKAGE);
builder.setShortName(getSimpleNameIndex(namespaceDescriptor.getName()));
NamespaceDescriptorParent containingDeclaration = namespaceDescriptor.getContainingDeclaration();
if (containingDeclaration instanceof NamespaceDescriptor) {
NamespaceDescriptor parentNamespace = (NamespaceDescriptor) containingDeclaration;
if (!DescriptorUtils.isRootNamespace(parentNamespace)) {
builder.setParentQualifiedName(getFqNameIndex(parentNamespace));
public int getFqNameIndex(@NotNull FqName fqName) {
int result = -1;
for (Name segment : fqName.pathSegments()) {
QualifiedName.Builder builder = QualifiedName.newBuilder();
builder.setShortName(getSimpleNameIndex(segment));
if (result != -1) {
builder.setParentQualifiedName(result);
}
result = qualifiedNames.intern(builder);
}
else {
builder.clearParentQualifiedName();
}
return qualifiedNames.intern(builder);
return result;
}
}
@@ -27,6 +27,10 @@ public abstract class SerializerExtension {
return true;
}
public void serializeCallable(@NotNull CallableMemberDescriptor callable, @NotNull ProtoBuf.Callable.Builder proto) {
public void serializeCallable(
@NotNull CallableMemberDescriptor callable,
@NotNull ProtoBuf.Callable.Builder proto,
@NotNull NameTable nameTable
) {
}
}
@@ -1,6 +1,7 @@
package org.jetbrains.jet.descriptors.serialization.descriptors;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.descriptors.serialization.NameResolver;
import org.jetbrains.jet.descriptors.serialization.ProtoBuf;
import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
import org.jetbrains.jet.lang.descriptors.ClassOrNamespaceDescriptor;
@@ -19,7 +20,10 @@ public interface AnnotationDeserializer {
@NotNull
@Override
public List<AnnotationDescriptor> loadCallableAnnotations(
@NotNull ClassOrNamespaceDescriptor container, @NotNull ProtoBuf.Callable callableProto
@NotNull ClassOrNamespaceDescriptor container,
@NotNull ProtoBuf.Callable proto,
@NotNull NameResolver nameResolver,
@NotNull AnnotatedCallableKind kind
) {
return notSupported();
}
@@ -30,18 +34,27 @@ public interface AnnotationDeserializer {
return notSupported();
}
@NotNull
private List<AnnotationDescriptor> notSupported() {
throw new UnsupportedOperationException("Annotations are not supported");
}
};
enum AnnotatedCallableKind {
FUNCTION,
PROPERTY_GETTER,
PROPERTY_SETTER
}
@NotNull
List<AnnotationDescriptor> loadClassAnnotations(@NotNull ClassDescriptor descriptor, @NotNull ProtoBuf.Class classProto);
@NotNull
List<AnnotationDescriptor> loadCallableAnnotations(
@NotNull ClassOrNamespaceDescriptor container,
@NotNull ProtoBuf.Callable callableProto
@NotNull ProtoBuf.Callable proto,
@NotNull NameResolver nameResolver,
@NotNull AnnotatedCallableKind kind
);
@NotNull
@@ -76,7 +76,10 @@ public abstract class AbstractDescriptorSerializationTest extends KotlinTestWith
@NotNull
@Override
public List<AnnotationDescriptor> loadCallableAnnotations(
@NotNull ClassOrNamespaceDescriptor container, @NotNull ProtoBuf.Callable callableProto
@NotNull ClassOrNamespaceDescriptor container,
@NotNull ProtoBuf.Callable proto,
@NotNull NameResolver nameResolver,
@NotNull AnnotatedCallableKind kind
) {
throw new UnsupportedOperationException(); // TODO
}