Protobuf: refactored generateSerializationCode() in FieldGenerator + some other little refactoring

This commit is contained in:
dsavvinov
2016-07-20 19:56:40 +03:00
parent 7f90be6c0a
commit 33836aeafb
8 changed files with 275 additions and 221 deletions
+33 -30
View File
@@ -22,38 +22,41 @@ In folder /src/ you can find sources for ProtoKot-runtime, that will be used by
## Using generated code
Example:
// Messages work only with CodedStream classes, provided by ProtoKot-runtime library.
// One can create CodedStream passing any instance of corresponding Stream from Java's library.
val s = ByteArrayOutputStream()
val outs = CodedOutputStream(s)
// All messages are immutable. Use Builders for creating new messages
val msg = Person.BuilderPerson()
.setEmail("wtf@dasda.com") // all setters return this builder, so you could chain modifiers in LINQ-style
.setId(42)
.setName("John Doe")
.setPhones(arrayOf( // repeated fields stored as Array<>, so use arrayOf() for creating repeated fields
Person.PhoneNumber.BuilderPhoneNumber()
.setNumber("342143-23423-42")
.setType(Person.PhoneType.HOME)
.build()
))
.build() // don't forget to call build() to produce message
msg.writeTo(outs)
```java
// Messages work only with CodedStream classes, provided by ProtoKot-runtime library.
// One can create CodedStream passing any instance of corresponding Stream from Java's library.
val s = ByteArrayOutputStream()
val outs = CodedOutputStream(s)
// Now let's use output stream as input to read our message from it!
var ins = CodedInputStream(ByteArrayInputStream(s.toByteArray()))
// All messages are immutable. Use Builders for creating new messages
val msg = Person.BuilderPerson()
.setEmail("wtf@dasda.com") // all setters return this builder, so you could chain modifiers in LINQ-style
.setId(42)
.setName("John Doe")
.setPhones(arrayOf( // repeated fields stored as Array<>, so use arrayOf() for creating repeated fields
Person.PhoneNumber.BuilderPhoneNumber()
.setNumber("342143-23423-42")
.setType(Person.PhoneType.HOME)
.build()
))
.build() // don't forget to call build() to produce message
msg.writeTo(outs)
// Create default instance of message
var readMsg = Person.BuilderPerson().build()
// Read in that message data from input stream.
readMsg.mergeFrom(ins)
// Now let's use output stream as input to read our message from it!
var ins = CodedInputStream(ByteArrayInputStream(s.toByteArray()))
// Note, that currently mergeFrom is the only way to mutate instance of message.
// Don't rely on it, probably mergeFrom will be refactored lately to guarantee full immutability of mesages.
// Create default instance of message
var readMsg = Person.BuilderPerson().build()
// Read in that message data from input stream.
readMsg.mergeFrom(ins)
// Note, that currently mergeFrom is the only way to mutate instance of message.
// Don't rely on it, probably mergeFrom will be refactored lately to guarantee full immutability of mesages.
// Better way to read a message:
ins = CodedInputStream(ByteArrayInputStream(s.toByteArray()))
readMsg = Person.BuilderPerson().readFrom(ins).build()
assert(msg == readMsg)
```
// Better way to read a message:
ins = CodedInputStream(ByteArrayInputStream(s.toByteArray()))
readMsg = Person.BuilderPerson().readFrom(ins).build()
assert(msg == readMsg)
@@ -97,7 +97,7 @@ ClassGenerator::ClassGenerator(Descriptor const *descriptor, NameResolver * name
for (int i = 0; i < enums_declarations_count; ++i) {
EnumDescriptor const * nestedEnumDescriptor = descriptor->enum_type(i);
nameResolver->addClass(nestedEnumDescriptor->name(), getFullType());
enumsDeclaraions.push_back(new EnumGenerator(nestedEnumDescriptor));
enumsDeclaraions.push_back(new EnumGenerator(nestedEnumDescriptor, nameResolver));
}
/**
@@ -275,7 +275,7 @@ void ClassGenerator::generateParseMethods(io::Printer *printer) const {
// messages are not required to end with 0-tag, therefore parsing method should check for EOF
printer->Print("if (input.isAtEnd()) { return false }\n");
// read tag and check if some field will follow (0-tag inidcates end of message)
// read tag and check if some field will follow (0-tag indicates end of message)
printer->Print("val tag = input.readInt32NoTag()\n");
printer->Print("if (tag == 0) { return false } \n");
@@ -308,6 +308,8 @@ void ClassGenerator::generateParseMethods(io::Printer *printer) const {
}
}
// TODO: add parsing of unknown fields
printer->Outdent();
printer->Print("}\n"); // when-clause
@@ -19,6 +19,7 @@ namespace kotlin {
class FieldGenerator; // declared in "kotlin_file_generator.h"
class NameResolver; // declared in "kotlin_name_resolver.h"
class EnumGenerator; // declared in "kotlin_enum_generator.h"
class ClassGenerator {
public:
@@ -41,6 +42,7 @@ private:
void generateBuilder (io::Printer * printer) const;
void generateBuildMethod (io::Printer * printer) const;
void generateInitSection (io::Printer * printer) const;
/**
* Flag isBuilder used for reducing code repeating, as code for class itself
* and for its inner builder are structurally very alike and can be generated
@@ -48,12 +50,6 @@ private:
*/
void generateHeader(io::Printer * printer, bool isBuilder = false) const;
/**
* IsRead flag indicates that readFrom method should be generated, otherwise
* writeTo method is generated. Motivation is similar to the isBuilder flag:
* both methods are structurally the same with some trivial substitutions
* (read -> write and etc.)
*/
void generateWriteToMethod(io::Printer *printer) const;
void generateMergeMethods(io::Printer *printer) const;
void generateParseMethods(io::Printer * printer) const;
@@ -54,13 +54,14 @@ void EnumGenerator::generateCode(io::Printer * printer) const {
}
printer->Print("\n");
gemerateEnumConverters(printer);
generateEnumConverter(printer);
printer->Outdent();
printer->Print("}");
}
void EnumGenerator::gemerateEnumConverters(io::Printer *printer) const {
void EnumGenerator::generateEnumConverter(io::Printer *printer) const {
// note that full-qualification is not necessary as this code resides in enum namespace
map <string, string> vars;
vars["dollar"] = "$";
vars["type"] = simpleName;
@@ -105,14 +106,21 @@ EnumGenerator::~EnumGenerator() {
}
}
EnumGenerator::EnumGenerator(EnumDescriptor const *descriptor) {
simpleName = descriptor->name();
EnumGenerator::EnumGenerator(EnumDescriptor const *descriptor, NameResolver * nameResolver)
: simpleName(descriptor->name())
, nameResolver(nameResolver)
{
int values_count = descriptor->value_count();
for (int i = 0; i < values_count; ++i) {
enumValues.push_back(new EnumValueGenerator(descriptor->value(i)));
}
}
string EnumGenerator::getFullType() const {
return nameResolver->getClassName(simpleName);
}
} // namespace kotlin
} // namespace compiler
} // namespace protobuf
@@ -7,6 +7,9 @@
#include <google/protobuf/io/printer.h>
#include <google/protobuf/descriptor.h>
#include "kotlin_name_resolver.h"
class NameResolver; // declared in "kotlin_name_resolver.h"
namespace google {
namespace protobuf {
@@ -24,15 +27,19 @@ public:
class EnumGenerator {
public:
EnumGenerator(EnumDescriptor const * descriptor);
EnumGenerator(EnumDescriptor const * descriptor, NameResolver * nameResolver);
~EnumGenerator();
string simpleName;
vector <EnumValueGenerator *> enumValues;
/* Return full-qualified name of enum */
string getFullType() const;
void generateCode(io::Printer *) const;
private:
void gemerateEnumConverters(io::Printer *printer) const;
void generateEnumConverter(io::Printer *printer) const;
NameResolver * nameResolver;
};
} // namespace kotlin
@@ -58,191 +58,224 @@ FieldGenerator::FieldGenerator(FieldDescriptor const * descriptor, ClassGenerato
, protoLabel(descriptor->label())
{ }
// TODO: long, complicated and messy method. Refactor it ASAP
void FieldGenerator::generateSerializationCode(io::Printer *printer, bool isRead, bool noTag) const {
void FieldGenerator::generateSerializationForRepeated(io::Printer * printer, bool isRead, bool noTag) const {
map <string, string> vars;
vars["type"] = getKotlinFunctionSuffix() + (noTag ? "NoTag" : "");
vars["fieldNumber"] = std::to_string(getFieldNumber());
vars["maybeFieldNumber"] = noTag ? "" : std::to_string(getFieldNumber());
vars["builderType"] = getUnderlyingTypeGenerator().getFullType();
vars["initValue"] = getUnderlyingTypeGenerator().getInitValue();
vars["fieldName"] = simpleName;
vars["arg"] = isRead ? "input" : "output";
vars["maybeComma"] = ", ";
/**
* First of all, try to generate syntax for repeated fields because it's separate case.
* Do this according to protobuf format:
* - Check if size of array is > 0, because empty repeated fields shouldn't appear in message
* - Write tag explicitly
* - Write length as int32 (note that tag shouldn't be added)
* - Write all repeated elements via recursive call (again, without tags)
*/
if (getProtoLabel() == FieldDescriptor::LABEL_REPEATED) {
// tag
if (isRead) {
if (!noTag) {
printer->Print(vars, "val tag = input.readTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
}
printer->Print(vars, "val expectedSize = input.readInt32NoTag()\n");
printer->Print("var readSize = 0\n");
printer->Print(vars, "while(readSize != expectedSize) {\n");
printer->Indent();
/* hack: copy current FieldGenerator and change label to OPTIONAL. Also change name to
name of iterator in for-loop.
This will allow to re-use this function for generating serialization code for elements of array.
More importantly, this will care about nested types too.
Efficiently, it inlines serialization code for all underlying types.
This hack isn't necessary from the architectural point of view and could be safely
removed as soon as target code will support inheritance and interfaces.
(then writing CodedOutputStream.writeMessage will be possible).
*/
FieldGenerator singleFieldGen = getUnderlyingTypeGenerator();
/* Another dirty hack here: create tmp variable of a given type and read it from input stream
then add that tmp var into list.
This is made because simple recursive call will generate code that tries to array[i].mergeFrom().
This is incorrect because array has old size, while 'i' iterates over new size, which can lead
to ArrayOutOfIndex errors.
*/
// TODO: stub here, resolve name properly!
vars["builderType"] = getUnderlyingTypeGenerator().getFullType();
vars["initValue"] = getUnderlyingTypeGenerator().getInitValue();
printer->Print(vars, "var tmp: $builderType$ = $initValue$\n");
singleFieldGen.simpleName = "tmp";
singleFieldGen.protoLabel = FieldDescriptor::LABEL_OPTIONAL;
// Note that primitive types are packed by default in proto3, i.e. they are should be written without tag
bool isPrimitive = descriptor->type() != FieldDescriptor::TYPE_BYTES &&
descriptor->type() != FieldDescriptor::TYPE_MESSAGE &&
descriptor->type() != FieldDescriptor::TYPE_STRING &&
descriptor->type() != FieldDescriptor::TYPE_ENUM;
singleFieldGen.generateSerializationCode(printer, isRead, /* noTag = */ isPrimitive);
singleFieldGen.generateSizeEstimationCode(printer, /* varName = */ "readSize"); // add size of current element to total size
printer->Print(vars, "$fieldName$.add(tmp)\n");
printer->Outdent();
printer->Print("}\n");
}
else {
printer->Print(vars, "if ($fieldName$.size > 0) {\n");
printer->Indent();
// tag
printer->Print(vars, "output.writeTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
// length
printer->Print(vars, "var arrayByteSize = 0\n");
generateSizeEstimationCode(printer, "arrayByteSize", /* noTag = */ true);
printer->Print(vars, "output.writeInt32NoTag(arrayByteSize)\n");
// all elements
printer->Print(vars, "for (item in $fieldName$) {\n");
printer->Indent();
// hack: see above
FieldGenerator singleFieldGen = FieldGenerator(descriptor, enclosingClass, nameResolver);
singleFieldGen.simpleName = "item";
singleFieldGen.protoLabel = FieldDescriptor::LABEL_OPTIONAL;
// TODO: maybe refactor this in name_resolving or separate method at least
// Note that primitive types are packed by default in proto3, i.e. they are should be written without tag
bool isPrimitive = descriptor->type() != FieldDescriptor::TYPE_BYTES &&
descriptor->type() != FieldDescriptor::TYPE_MESSAGE &&
descriptor->type() != FieldDescriptor::TYPE_STRING &&
descriptor->type() != FieldDescriptor::TYPE_ENUM;
singleFieldGen.generateSerializationCode(printer, isRead, /* noTag = */ isPrimitive);
printer->Outdent(); // for-loop
printer->Print("}\n");
printer->Outdent(); // if-clause
printer->Print("}\n");
}
return;
}
/*
Then check for conversions 'int -> enum-value' and \enum-value -> int' if current
field is enum.
This is necessary, because CodedStream stores enums as Ints in wire, delegating
responsibility for casting those Ints to enum values and vice versa to the caller.
Example: enumField = fromIntToMyEnumName(input.readEnum(42))
Example: output.writeEnum(42, enumField.ord)
*/
if (descriptor->type() == FieldDescriptor::TYPE_ENUM) {
vars["converter"] = getEnumFromIntConverter();
if (isRead) {
printer->Print(vars, "$fieldName$ = $converter$(input.read$type$($maybeFieldNumber$))\n");
}
else {
printer->Print(vars, "output.write$type$ ($maybeFieldNumber$$maybeComma$$fieldName$.ord)\n");
}
return;
}
/*
Then check for nested messages. Then we re-use writeTo method, that should be defined in
that message.
Note that readFrom/writeTo methods write message as it's top-level message, i.e. without
any tags. Therefore, we have to prepend tags and size manually.
*/
if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE) {
if (isRead) {
vars["fieldNumber"] = std::to_string(getFieldNumber());
vars["dollar"] = "$";
// We will create some temporary variables
// So we place following code into separate block for the sake of hygiene
printer->Print("run {\n");
printer->Indent();
// read tag
if (!noTag) {
printer->Print(vars, "input.readTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
}
// read expected size
printer->Print(vars, "val expectedSize = input.readInt32NoTag()\n");
// read message itself without tag
printer->Print(vars,
"$fieldName$.mergeFromWithSize(input, expectedSize)\n");
// check that actual size equal to expected size
printer->Print(vars, "if (expectedSize != $fieldName$.getSize()) { "
"throw InvalidProtocolBufferException ("
"\"Expected size $dollar${expectedSize} got $dollar${$fieldName$.getSize()}"
"\") }\n");
printer->Outdent();
printer->Print("}\n");
}
else {
vars["fieldNumber"] = std::to_string(getFieldNumber());
// write tag
printer->Print(vars, "output.writeTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
// write message length via runtime-call
printer->Print(vars, "output.writeInt32NoTag($fieldName$.getSize())\n");
// write message itself without tag
printer->Print(vars,
"$fieldName$.writeTo(output)\n");
}
return;
}
/* Finally, serialize trivial cases */
if (isRead) {
printer->Print(vars, "$fieldName$ = input.read$type$($maybeFieldNumber$)\n");
if (!noTag) {
printer->Print(vars, "val tag = input.readTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
}
printer->Print(vars, "val expectedSize = input.readInt32NoTag()\n");
printer->Print("var readSize = 0\n");
printer->Print(vars, "while(readSize != expectedSize) {\n");
printer->Indent();
/* hack: copy current FieldGenerator and change label to OPTIONAL. Also change name to
name of iterator in for-loop.
This will allow to re-use this function for generating serialization code for elements of array.
More importantly, this will care about nested types too.
Efficiently, it inlines serialization code for all underlying types.
This hack isn't necessary from the architectural point of view and could be safely
removed as soon as target code will support inheritance and interfaces.
(then writing CodedOutputStream.writeMessage will be possible).
*/
FieldGenerator singleFieldGen = getUnderlyingTypeGenerator();
/* Another dirty hack here: create tmp variable of a given type and read it from input stream
then add that tmp var into list.
This is made because simple recursive call will generate code that tries to array[i].mergeFrom().
This is incorrect because array has old size, while 'i' iterates over new size, which can lead
to ArrayOutOfIndex errors.
*/
printer->Print(vars, "var tmp: $builderType$ = $initValue$\n");
singleFieldGen.simpleName = "tmp";
singleFieldGen.protoLabel = FieldDescriptor::LABEL_OPTIONAL;
// Note that primitive types are packed by default in proto3, i.e. they are should be written without tag
bool isPrimitive = descriptor->type() != FieldDescriptor::TYPE_BYTES &&
descriptor->type() != FieldDescriptor::TYPE_MESSAGE &&
descriptor->type() != FieldDescriptor::TYPE_STRING &&
descriptor->type() != FieldDescriptor::TYPE_ENUM;
singleFieldGen.generateSerializationCode(printer, isRead, /* noTag = */ isPrimitive);
singleFieldGen.generateSizeEstimationCode(printer, /* varName = */
"readSize"); // add size of current element to total size
printer->Print(vars, "$fieldName$.add(tmp)\n");
printer->Outdent();
printer->Print("}\n");
}
else {
printer->Print(vars, "output.write$type$ ($maybeFieldNumber$$maybeComma$$fieldName$)\n");
/**
* Protobuf format:
* - Check if size of array is > 0, because empty repeated fields shouldn't appear in message
* - Write tag explicitly
* - Write length as int32 (note that tag shouldn't be added)
* - Write all repeated elements via recursive call (for primitive types without tags)
*/
printer->Print(vars, "if ($fieldName$.size > 0) {\n");
printer->Indent();
// tag
printer->Print(vars, "output.writeTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
// length
printer->Print(vars, "var arrayByteSize = 0\n");
generateSizeEstimationCode(printer, "arrayByteSize", /* noTag = */ true);
printer->Print(vars, "output.writeInt32NoTag(arrayByteSize)\n");
// all elements
printer->Print(vars, "for (item in $fieldName$) {\n");
printer->Indent();
// hack: see above
FieldGenerator singleFieldGen = FieldGenerator(descriptor, enclosingClass, nameResolver);
singleFieldGen.simpleName = "item";
singleFieldGen.protoLabel = FieldDescriptor::LABEL_OPTIONAL;
// TODO: maybe refactor this in name_resolving or separate method at least
bool isPrimitive = descriptor->type() != FieldDescriptor::TYPE_BYTES &&
descriptor->type() != FieldDescriptor::TYPE_MESSAGE &&
descriptor->type() != FieldDescriptor::TYPE_STRING &&
descriptor->type() != FieldDescriptor::TYPE_ENUM;
singleFieldGen.generateSerializationCode(printer, isRead, /* noTag = */ isPrimitive);
printer->Outdent(); // for-loop
printer->Print("}\n");
printer->Outdent(); // if-clause
printer->Print("}\n");
}
}
void FieldGenerator::generateSerializationForEnums(io::Printer * printer, bool isRead, bool noTag) const {
map <string, string> vars;
vars["converter"] = getEnumFromIntConverter();
vars["fieldName"] = simpleName;
vars["suffix"] = getKotlinFunctionSuffix();
vars["fieldNumber"] = std::to_string(getFieldNumber());
if (isRead) {
if (noTag) {
printer->Print(vars, "$fieldName$ = $converter$(input.read$suffix$NoTag())\n");
}
else {
printer->Print(vars, "$fieldName$ = $converter$(input.read$suffix$($fieldNumber$))\n");
}
}
else {
if (noTag) {
printer->Print(vars, "output.write$suffix$NoTag ()\n");
}
else {
printer->Print(vars, "output.write$suffix$ ($fieldNumber$, $fieldName$.ord)\n");
}
}
}
void FieldGenerator::generateSerializationForMessages(io::Printer * printer, bool isRead, bool noTag) const {
map <string, string> vars;
vars["fieldNumber"] = std::to_string(getFieldNumber());
vars["dollar"] = "$";
vars["fieldName"] = simpleName;
if (isRead) {
// We will create some temporary variables
// So we place following code into separate block for the sake of hygiene
printer->Print("run {\n");
printer->Indent();
// read tag
if (!noTag) {
printer->Print(vars, "input.readTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
}
// read expected size
printer->Print(vars, "val expectedSize = input.readInt32NoTag()\n");
// TODO: think about it, as it's not good approach - if some error occurs, we will read more bytes
// than expectedSize from CodedInputStream. That could potentially lead to some lingering problems in wire.
// read message itself without tag, but limiting its size to expectedSize
printer->Print(vars,
"$fieldName$.mergeFromWithSize(input, expectedSize)\n");
// check that actual size equal to expected size
printer->Print(vars, "if (expectedSize != $fieldName$.getSize()) { "
"throw InvalidProtocolBufferException ("
"\"Expected size $dollar${expectedSize} got $dollar${$fieldName$.getSize()}"
"\") }\n");
printer->Outdent();
printer->Print("}\n");
}
else {
// write tag
printer->Print(vars, "output.writeTag($fieldNumber$, WireType.LENGTH_DELIMITED)\n");
// write message length
printer->Print(vars, "output.writeInt32NoTag($fieldName$.getSize())\n");
// write message itself without tag
printer->Print(vars,
"$fieldName$.writeTo(output)\n");
}
}
void FieldGenerator::generateSerializationForPrimitives(io::Printer * printer, bool isRead, bool noTag) const {
map <string, string> vars;
vars["fieldName"] = simpleName;
vars["suffix"] = getKotlinFunctionSuffix();
vars["fieldNumber"] = std::to_string(getFieldNumber());
if (isRead) {
if (noTag) {
printer->Print(vars, "$fieldName$ = input.read$suffix$NoTag()\n");
}
else {
printer->Print(vars, "$fieldName$ = input.read$suffix$ ($fieldNumber$)");
}
}
else {
if (noTag) {
printer->Print(vars, "output.write$suffix$NoTag ()\n");
}
else {
printer->Print(vars, "output.write$suffix$ ($fieldNumber$, $fieldName$)\n");
}
}
}
void FieldGenerator::generateSerializationCode(io::Printer *printer, bool isRead, bool noTag) const {
/* Try to generate syntax for serialization of repeated fields.
* Note that it should be first check because of Google's FieldDescriptor structure */
if (getProtoLabel() == FieldDescriptor::LABEL_REPEATED) {
generateSerializationForRepeated(printer, isRead, noTag);
return;
}
/* Then check is current field is enum. We have to handle it separately too, because
* we have to pass enums as Int's to CodedStreams as per protobuf-format */
if (descriptor->type() == FieldDescriptor::TYPE_ENUM) {
generateSerializationForEnums(printer, isRead, noTag);
return;
}
/* Then check for nested messages. Here we re-use writeTo method, that should be defined in
* that message.
* Note that readFrom/writeTo methods write message as it's top-level message, i.e. without
* any tags. Therefore, we have to prepend tags and size manually. */
if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE) {
generateSerializationForMessages(printer, isRead, noTag);
return;
}
/* Finally, serialize trivial cases */
generateSerializationForPrimitives(printer, isRead, noTag);
}
void FieldGenerator::generateSetter(io::Printer *printer) const {
map <string, string> vars;
@@ -265,7 +298,7 @@ void FieldGenerator::generateRepeatedMethods(io::Printer * printer, bool isBuild
vars["elementType"] = getUnderlyingTypeGenerator().getSimpleType();
vars["arg"] = "value";
vars["fieldName"] = simpleName;
vars["builderName"] = enclosingClass->getBuilderFullType(); // TODO: call to non-existent field in map.
vars["builderName"] = enclosingClass->getBuilderFullType();
// generate indexed setter for builders
if (isBuilder) {
@@ -26,6 +26,11 @@ private:
void generateSetter(io::Printer * printer) const;
void generateRepeatedMethods(io::Printer * printer, bool isBuilder) const;
void generateSerializationForRepeated (io::Printer * printer, bool isRead, bool noTag) const;
void generateSerializationForEnums (io::Printer * printer, bool isRead, bool noTag) const;
void generateSerializationForMessages (io::Printer * printer, bool isRead, bool noTag) const;
void generateSerializationForPrimitives (io::Printer * printer, bool isRead, bool noTag) const;
public:
ClassGenerator const * enclosingClass; // class, in which that field is defined
NameResolver * nameResolver;