Move essential metadata-related classes to 'metadata' and 'metadata.jvm'

This commit is contained in:
Alexander Udalov
2018-03-02 19:33:10 +01:00
parent 88dea9aa70
commit 3f3e05a9d2
55 changed files with 112 additions and 120 deletions
@@ -0,0 +1,264 @@
/*
* Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.metadata.jvm.deserialization;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
import static org.jetbrains.kotlin.metadata.jvm.deserialization.UtfEncodingKt.MAX_UTF8_INFO_LENGTH;
public class BitEncoding {
private static final boolean FORCE_8TO7_ENCODING = "true".equals(System.getProperty("kotlin.jvm.serialization.use8to7"));
private static final char _8TO7_MODE_MARKER = (char) -1;
private BitEncoding() {
}
/**
* Converts a byte array of serialized data to an array of {@code String} satisfying JVM annotation value argument restrictions:
* <ol>
* <li>Each string's length should be no more than 65535</li>
* <li>UTF-8 representation of each string cannot contain bytes in the range 0xf0..0xff</li>
* </ol>
*/
@NotNull
public static String[] encodeBytes(@NotNull byte[] data) {
// TODO: try both encodings here and choose the best one (with the smallest size)
if (!FORCE_8TO7_ENCODING) {
return UtfEncodingKt.bytesToStrings(data);
}
byte[] bytes = encode8to7(data);
// Since 0x0 byte is encoded as two bytes in the Modified UTF-8 (0xc0 0x80) and zero is rather common to byte arrays, we increment
// every byte by one modulo max byte value, so that the less common value 0x7f will be represented as two bytes instead.
addModuloByte(bytes, 1);
return splitBytesToStringArray(bytes);
}
/**
* Converts a byte array to another byte array, every element of which is in the range 0x0..0x7f.
*
* The conversion is equivalent to the following: input bytes are combined into one long bit string. This big string is then split into
* groups of 7 bits. Each resulting 7-bit chunk is then converted to a byte (with a leading bit = 0). The last chunk may have less than
* 7 bits, it's prepended with zeros to form a byte. The result is then the array of these bytes, each of which is obviously in the
* range 0x0..0x7f.
*
* Suppose the input of 4 bytes is given (bytes are listed from the beginning to the end, each byte from the least significant bit to
* the most significant bit, bits within each byte are numbered):
*
* 01234567 01234567 01234567 01234567
*
* The output for this kind of input will be of the following form ('#' represents a zero bit):
*
* 0123456# 7012345# 6701234# 5670123# 4567####
*/
@NotNull
private static byte[] encode8to7(@NotNull byte[] data) {
// ceil(data.length * 8 / 7)
int resultLength = (data.length * 8 + 6) / 7;
byte[] result = new byte[resultLength];
// We maintain a pointer to the bit in the input, which is represented by two numbers: index of the current byte in the input and
// the index of a bit inside this byte (0 is least significant, 7 is most significant)
int byteIndex = 0;
int bit = 0;
// Write all resulting bytes except the last one. To do this we need to collect exactly 7 bits, starting from the current, into a
// byte. In almost all cases these 7 bits can be collected from two parts: the first is several (at least one) most significant bits
// from the current byte, the second is several (maybe zero) least significant bits from the next byte. The special case is when the
// current bit is the first (least significant) bit in its byte (bit == 0): then the 7 needed bits are just the 7 least significant
// of the current byte.
for (int i = 0; i < resultLength - 1; i++) {
if (bit == 0) {
result[i] = (byte) (data[byteIndex] & 0x7f);
bit = 7;
continue;
}
int firstPart = (data[byteIndex] & 0xff) >>> bit;
int newBit = (bit + 7) & 7;
int secondPart = (data[++byteIndex] & ((1 << newBit) - 1)) << 8 - bit;
result[i] = (byte) (firstPart + secondPart);
bit = newBit;
}
// Write the last byte, which is just several most significant bits of the last byte in the input, padded with zeros
if (resultLength > 0) {
assert bit != 0 : "The last chunk cannot start from the input byte since otherwise at least one bit will remain unprocessed";
assert byteIndex == data.length - 1 : "The last 7-bit chunk should be encoded from the last input byte: " +
byteIndex + " != " + (data.length - 1);
result[resultLength - 1] = (byte) ((data[byteIndex] & 0xff) >>> bit);
}
return result;
}
private static void addModuloByte(@NotNull byte[] data, int increment) {
for (int i = 0, n = data.length; i < n; i++) {
data[i] = (byte) ((data[i] + increment) & 0x7f);
}
}
/**
* Converts a big byte array into the array of strings, where each string, when written to the constant pool table in bytecode, produces
* a byte array of not more than MAX_UTF8_INFO_LENGTH. Each byte, except those which are 0x0, occupies exactly one byte in the constant
* pool table. Zero bytes occupy two bytes in the table each.
*
* When strings are constructed from the array of bytes here, they are encoded in the platform's default encoding. This is fine: the
* conversion to the Modified UTF-8 (which here would be equivalent to replacing each 0x0 with 0xc0 0x80) will happen later by ASM, when
* it writes these strings to the bytecode
*/
@NotNull
private static String[] splitBytesToStringArray(@NotNull byte[] data) {
List<String> result = new ArrayList<String>();
// The offset where the currently processed string starts
int off = 0;
// The effective length the bytes of the current string would occupy in the constant pool table.
// 2 because the first char is -1 which denotes the encoding mode and occupies two bytes in Modified UTF-8
int len = 2;
boolean encodingModeAdded = false;
for (int i = 0, n = data.length; i < n; i++) {
// When the effective length reaches at least MAX - 1, we add the current string to the result. Note that the effective length
// is at most MAX here: non-zero bytes occupy 1 byte and zero bytes occupy 2 bytes, so we couldn't jump over more than one byte
if (len >= MAX_UTF8_INFO_LENGTH - 1) {
assert len <= MAX_UTF8_INFO_LENGTH : "Produced strings cannot contain more than " + MAX_UTF8_INFO_LENGTH + " bytes: " + len;
String string = new String(data, off, i - off);
if (!encodingModeAdded) {
encodingModeAdded = true;
result.add(_8TO7_MODE_MARKER + string);
}
else {
result.add(string);
}
off = i;
len = 0;
}
if (data[i] == 0) {
len += 2;
}
else {
len++;
}
}
if (len >= 0) {
result.add(new String(data, off, data.length - off));
}
//noinspection SSBasedInspection
return result.toArray(new String[result.size()]);
}
/**
* Converts encoded array of {@code String} obtained by {@link BitEncoding#encodeBytes(byte[])} back to a byte array.
*/
@NotNull
public static byte[] decodeBytes(@NotNull String[] data) {
if (data.length > 0 && !data[0].isEmpty()) {
char possibleMarker = data[0].charAt(0);
if (possibleMarker == UtfEncodingKt.UTF8_MODE_MARKER) {
return UtfEncodingKt.stringsToBytes(dropMarker(data));
}
if (possibleMarker == _8TO7_MODE_MARKER) {
data = dropMarker(data);
}
}
byte[] bytes = combineStringArrayIntoBytes(data);
// Adding 0x7f modulo max byte value is equivalent to subtracting 1 the same modulo, which is inverse to what happens in encodeBytes
addModuloByte(bytes, 0x7f);
return decode7to8(bytes);
}
@NotNull
private static String[] dropMarker(@NotNull String[] data) {
// Clone because the clients should be able to use the passed array for their own purposes.
// This is cheap because the size of the array is 1 or 2 almost always.
String[] result = data.clone();
result[0] = result[0].substring(1);
return result;
}
/**
* Combines the array of strings resulted from encodeBytes() into one long byte array
*/
@NotNull
private static byte[] combineStringArrayIntoBytes(@NotNull String[] data) {
int resultLength = 0;
for (String s : data) {
assert s.length() <= MAX_UTF8_INFO_LENGTH : "String is too long: " + s.length();
resultLength += s.length();
}
byte[] result = new byte[resultLength];
int p = 0;
for (String s : data) {
for (int i = 0, n = s.length(); i < n; i++) {
result[p++] = (byte) s.charAt(i);
}
}
return result;
}
/**
* Decodes the byte array resulted from encode8to7().
*
* Each byte of the input array has at most 7 valuable bits of information. So the decoding is equivalent to the following: least
* significant 7 bits of all input bytes are combined into one long bit string. This bit string is then split into groups of 8 bits,
* each of which forms a byte in the output. If there are any leftovers, they are ignored, since they were added just as a padding and
* do not comprise a full byte.
*
* Suppose the following encoded byte array is given (bits are numbered the same way as in encode8to7() doc):
*
* 01234567 01234567 01234567 01234567
*
* The output of the following form would be produced:
*
* 01234560 12345601 23456012
*
* Note how all most significant bits and leftovers are dropped, since they don't contain any useful information
*/
@NotNull
private static byte[] decode7to8(@NotNull byte[] data) {
// floor(7 * data.length / 8)
int resultLength = 7 * data.length / 8;
byte[] result = new byte[resultLength];
// We maintain a pointer to an input bit in the same fashion as in encode8to7(): it's represented as two numbers: index of the
// current byte in the input and index of the bit in the byte
int byteIndex = 0;
int bit = 0;
// A resulting byte is comprised of 8 bits, starting from the current bit. Since each input byte only "contains 7 bytes", a
// resulting byte always consists of two parts: several most significant bits of the current byte and several least significant bits
// of the next byte
for (int i = 0; i < resultLength; i++) {
int firstPart = (data[byteIndex] & 0xff) >>> bit;
byteIndex++;
int secondPart = (data[byteIndex] & ((1 << (bit + 1)) - 1)) << 7 - bit;
result[i] = (byte) (firstPart + secondPart);
if (bit == 6) {
byteIndex++;
bit = 0;
}
else {
bit++;
}
}
return result;
}
}
@@ -0,0 +1,78 @@
/*
* Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.metadata.jvm.deserialization
// The purpose of this class is to map Kotlin classes to JVM bytecode desc strings, as KotlinTypeMapper does in the backend.
// It's used as an optimization during serialization/deserialization: if there's no JVM signature for a method/property/constructor,
// it means that the JVM signature should be trivially computable from the Kotlin signature with this class.
// It's not required to support everything in KotlinTypeMapper, but the more it does, the more we save on JVM signatures in proto metadata.
//
// WARNING: improving the behavior of this class MAY BREAK BINARY COMPATIBILITY of code compiled by Kotlin, because it may make
// the new compiler skip writing the signatures it now thinks are trivial, and the old compiler would recreate them incorrectly.
object ClassMapperLite {
// Kotlin ClassId -> JVM desc
// e.g. "kotlin.IntArray" -> "[I"
// "kotlin.String.Companion" -> "Lkotlin/jvm/internal/StringCompanionObject"
// "kotlin/collections/Map.Entry" -> "Ljava/util/Map$Entry"
private val map: Map<String, String> = mutableMapOf<String, String>().apply {
val primitives = listOf(
"Boolean", "Z",
"Char", "C",
"Byte", "B",
"Short", "S",
"Int", "I",
"Float", "F",
"Long", "J",
"Double", "D"
)
for (i in primitives.indices step 2) {
put("kotlin/${primitives[i]}", primitives[i + 1])
put("kotlin/${primitives[i]}Array", "[${primitives[i + 1]}")
}
put("kotlin/Unit", "V")
fun add(kotlinSimpleName: String, javaInternalName: String) {
put("kotlin/$kotlinSimpleName", "L$javaInternalName;")
}
add("Any", "java/lang/Object")
add("Nothing", "java/lang/Void")
add("Annotation", "java/lang/annotation/Annotation")
for (klass in listOf("String", "CharSequence", "Throwable", "Cloneable", "Number", "Comparable", "Enum")) {
add(klass, "java/lang/$klass")
}
for (klass in listOf("Iterator", "Collection", "List", "Set", "Map", "ListIterator")) {
add("collections/$klass", "java/util/$klass")
add("collections/Mutable$klass", "java/util/$klass")
}
add("collections/Iterable", "java/lang/Iterable")
add("collections/MutableIterable", "java/lang/Iterable")
add("collections/Map.Entry", "java/util/Map\$Entry")
add("collections/MutableMap.MutableEntry", "java/util/Map\$Entry")
for (i in 0..22) {
add("Function$i", "kotlin/jvm/functions/Function$i")
add("reflect/KFunction$i", "kotlin/reflect/KFunction")
}
for (klass in listOf("Char", "Byte", "Short", "Int", "Float", "Long", "Double", "String", "Enum")) {
add("$klass.Companion", "kotlin/jvm/internal/${klass}CompanionObject")
}
}
/**
* @param classId the name of the class in the format: "org/foo/bar/Test.Inner"
*/
@JvmStatic
fun mapClass(classId: String): String {
return map[classId] ?: "L${classId.replace('.', '$')};"
}
}
@@ -0,0 +1,108 @@
/*
* Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.metadata.jvm.deserialization
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.*
import org.jetbrains.kotlin.metadata.jvm.JvmProtoBuf
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import java.io.ByteArrayInputStream
import java.io.InputStream
object JvmProtoBufUtil {
val EXTENSION_REGISTRY: ExtensionRegistryLite = ExtensionRegistryLite.newInstance().apply(JvmProtoBuf::registerAllExtensions)
@JvmStatic
fun readClassDataFrom(data: Array<String>, strings: Array<String>): Pair<JvmNameResolver, ProtoBuf.Class> =
readClassDataFrom(BitEncoding.decodeBytes(data), strings)
@JvmStatic
fun readClassDataFrom(bytes: ByteArray, strings: Array<String>): Pair<JvmNameResolver, ProtoBuf.Class> {
val input = ByteArrayInputStream(bytes)
return Pair(input.readNameResolver(strings), ProtoBuf.Class.parseFrom(input, EXTENSION_REGISTRY))
}
@JvmStatic
fun readPackageDataFrom(data: Array<String>, strings: Array<String>): Pair<JvmNameResolver, ProtoBuf.Package> =
readPackageDataFrom(BitEncoding.decodeBytes(data), strings)
@JvmStatic
fun readPackageDataFrom(bytes: ByteArray, strings: Array<String>): Pair<JvmNameResolver, ProtoBuf.Package> {
val input = ByteArrayInputStream(bytes)
return Pair(input.readNameResolver(strings), ProtoBuf.Package.parseFrom(input, EXTENSION_REGISTRY))
}
@JvmStatic
fun readFunctionDataFrom(data: Array<String>, strings: Array<String>): Pair<JvmNameResolver, ProtoBuf.Function> {
val input = ByteArrayInputStream(BitEncoding.decodeBytes(data))
return Pair(input.readNameResolver(strings), ProtoBuf.Function.parseFrom(input, EXTENSION_REGISTRY))
}
private fun InputStream.readNameResolver(strings: Array<String>): JvmNameResolver =
JvmNameResolver(JvmProtoBuf.StringTableTypes.parseDelimitedFrom(this, EXTENSION_REGISTRY), strings)
// returns JVM signature in the format: "equals(Ljava/lang/Object;)Z"
fun getJvmMethodSignature(
proto: ProtoBuf.Function,
nameResolver: NameResolver,
typeTable: TypeTable
): String? {
val signature = proto.getExtensionOrNull(JvmProtoBuf.methodSignature)
val name = if (signature != null && signature.hasName()) signature.name else proto.name
val desc = if (signature != null && signature.hasDesc()) {
nameResolver.getString(signature.desc)
}
else {
val parameterTypes = listOfNotNull(proto.receiverType(typeTable)) + proto.valueParameterList.map { it.type(typeTable) }
val parametersDesc = parameterTypes.map { mapTypeDefault(it, nameResolver) ?: return null }
val returnTypeDesc = mapTypeDefault(proto.returnType(typeTable), nameResolver) ?: return null
parametersDesc.joinToString(separator = "", prefix = "(", postfix = ")") + returnTypeDesc
}
return nameResolver.getString(name) + desc
}
fun getJvmConstructorSignature(
proto: ProtoBuf.Constructor,
nameResolver: NameResolver,
typeTable: TypeTable
): String? {
val signature = proto.getExtensionOrNull(JvmProtoBuf.constructorSignature)
val desc = if (signature != null && signature.hasDesc()) {
nameResolver.getString(signature.desc)
}
else {
proto.valueParameterList.map {
mapTypeDefault(it.type(typeTable), nameResolver) ?: return null
}.joinToString(separator = "", prefix = "(", postfix = ")V")
}
return "<init>" + desc
}
fun getJvmFieldSignature(
proto: ProtoBuf.Property,
nameResolver: NameResolver,
typeTable: TypeTable
): PropertySignature? {
val signature = proto.getExtensionOrNull(JvmProtoBuf.propertySignature) ?: return null
val field =
if (signature.hasField()) signature.field else null
val name = if (field != null && field.hasName()) field.name else proto.name
val desc =
if (field != null && field.hasDesc()) nameResolver.getString(field.desc)
else mapTypeDefault(proto.returnType(typeTable), nameResolver) ?: return null
return PropertySignature(nameResolver.getString(name), desc)
}
data class PropertySignature(val name: String, val desc: String)
private fun mapTypeDefault(type: ProtoBuf.Type, nameResolver: NameResolver): String? {
return if (type.hasClassName()) ClassMapperLite.mapClass(nameResolver.getQualifiedClassName(type.className)) else null
}
}
@@ -0,0 +1,73 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.metadata.jvm.deserialization
import java.util.*
// The maximum possible length of the byte array in the CONSTANT_Utf8_info structure in the bytecode, as per JVMS7 4.4.7
const val MAX_UTF8_INFO_LENGTH = 65535
const val UTF8_MODE_MARKER = 0.toChar()
fun bytesToStrings(bytes: ByteArray): Array<String> {
val result = ArrayList<String>(1)
val buffer = StringBuilder()
var bytesInBuffer = 0
buffer.append(UTF8_MODE_MARKER)
// Zeros effectively occupy two bytes because each 0x0 is converted to 0x80 0xc0 in Modified UTF-8, see JVMS7 4.4.7
bytesInBuffer += 2
for (b in bytes) {
val c = b.toInt() and 0xFF // 0 <= c <= 255
buffer.append(c.toChar())
if (0 < b && b <= 127) {
bytesInBuffer++
}
else {
bytesInBuffer += 2
}
if (bytesInBuffer >= MAX_UTF8_INFO_LENGTH - 1) {
result.add(buffer.toString())
buffer.setLength(0)
bytesInBuffer = 0
}
}
if (!buffer.isEmpty()) {
result.add(buffer.toString())
}
return result.toTypedArray()
}
fun stringsToBytes(strings: Array<String>): ByteArray {
val resultLength = strings.sumBy { it.length }
val result = ByteArray(resultLength)
var i = 0
for (s in strings) {
for (si in 0..s.length - 1) {
result[i++] = s[si].toByte()
}
}
assert(i == result.size) { "Should have reached the end" }
return result
}