From 6dbd29e6a4f5dfc081df7e2a24599de8283bd46c Mon Sep 17 00:00:00 2001 From: Alexander Udalov Date: Tue, 12 Jan 2016 03:01:56 +0300 Subject: [PATCH] Support decoding of both old and new encoded metadata --- .../kotlin/serialization/jvm/BitEncoding.java | 40 ++++++++++++++++--- .../kotlin/serialization/jvm/utfEncoding.kt | 10 +++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/BitEncoding.java b/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/BitEncoding.java index 1d9eaf5a386..cf5da3eef4b 100644 --- a/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/BitEncoding.java +++ b/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/BitEncoding.java @@ -26,6 +26,8 @@ import static org.jetbrains.kotlin.serialization.jvm.UtfEncodingKt.MAX_UTF8_INFO public class BitEncoding { private static final boolean FORCE_8TO7_ENCODING = "true".equals(System.getProperty("kotlin.jvm.serialization.use8to7")); + private static final char _8TO7_MODE_MARKER = (char) -1; + private BitEncoding() { } @@ -129,15 +131,25 @@ public class BitEncoding { // The offset where the currently processed string starts int off = 0; - // The effective length the bytes of the current string would occupy in the constant pool table - int len = 0; + // The effective length the bytes of the current string would occupy in the constant pool table. + // 2 because the first char is -1 which denotes the encoding mode and occupies two bytes in Modified UTF-8 + int len = 2; + + boolean encodingModeAdded = false; for (int i = 0, n = data.length; i < n; i++) { // When the effective length reaches at least MAX - 1, we add the current string to the result. Note that the effective length // is at most MAX here: non-zero bytes occupy 1 byte and zero bytes occupy 2 bytes, so we couldn't jump over more than one byte if (len >= MAX_UTF8_INFO_LENGTH - 1) { assert len <= MAX_UTF8_INFO_LENGTH : "Produced strings cannot contain more than " + MAX_UTF8_INFO_LENGTH + " bytes: " + len; - result.add(new String(data, off, i - off)); + String string = new String(data, off, i - off); + if (!encodingModeAdded) { + encodingModeAdded = true; + result.add(_8TO7_MODE_MARKER + string); + } + else { + result.add(string); + } off = i; len = 0; } @@ -162,15 +174,31 @@ public class BitEncoding { */ @NotNull public static byte[] decodeBytes(@NotNull String[] data) { - if (!FORCE_8TO7_ENCODING) { - return UtfEncodingKt.stringsToBytes(data); + if (data.length > 0 && !data[0].isEmpty()) { + char possibleMarker = data[0].charAt(0); + if (possibleMarker == UtfEncodingKt.UTF8_MODE_MARKER) { + return UtfEncodingKt.stringsToBytes(dropMarker(data)); + } + if (possibleMarker == _8TO7_MODE_MARKER) { + data = dropMarker(data); + } } + byte[] bytes = combineStringArrayIntoBytes(data); // Adding 0x7f modulo max byte value is equivalent to subtracting 1 the same modulo, which is inverse to what happens in encodeBytes addModuloByte(bytes, 0x7f); return decode7to8(bytes); } + @NotNull + private static String[] dropMarker(@NotNull String[] data) { + // Clone because the clients should be able to use the passed array for their own purposes. + // This is cheap because the size of the array is 1 or 2 almost always. + String[] result = data.clone(); + result[0] = result[0].substring(1); + return result; + } + /** * Combines the array of strings resulted from encodeBytes() into one long byte array */ @@ -178,7 +206,7 @@ public class BitEncoding { private static byte[] combineStringArrayIntoBytes(@NotNull String[] data) { int resultLength = 0; for (String s : data) { - assert s.length() <= MAX_UTF8_INFO_LENGTH : "Too long string: " + s.length(); + assert s.length() <= MAX_UTF8_INFO_LENGTH : "String is too long: " + s.length(); resultLength += s.length(); } diff --git a/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/utfEncoding.kt b/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/utfEncoding.kt index 7a368fba2b0..90c7b4f3496 100644 --- a/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/utfEncoding.kt +++ b/core/descriptor.loader.java/src/org/jetbrains/kotlin/serialization/jvm/utfEncoding.kt @@ -21,6 +21,8 @@ import java.util.* // The maximum possible length of the byte array in the CONSTANT_Utf8_info structure in the bytecode, as per JVMS7 4.4.7 const val MAX_UTF8_INFO_LENGTH = 65535 +const val UTF8_MODE_MARKER = 0.toChar() + // Leading bytes are prefixed with 110 in UTF-8 private val LEADING_BYTE_MASK = 0b11000000 // Continuation bytes are prefixed with 10 in UTF-8 @@ -34,10 +36,16 @@ fun bytesToStrings(bytes: ByteArray): Array { val buffer = StringBuilder() var bytesInBuffer = 0 + buffer.append(UTF8_MODE_MARKER) + // Zeros effectively occupy two bytes because each 0x0 is converted to 0x80 0xc0 in Modified UTF-8, see JVMS7 4.4.7 + bytesInBuffer += 2 + for (b in bytes) { if (b >= 0) { buffer.append(b.toChar()) bytesInBuffer++ + // Zeros occupy two bytes + if (b == 0.toByte()) bytesInBuffer++ } else { val int = b.toInt() and 0xFF @@ -93,5 +101,7 @@ fun stringsToBytes(strings: Array): ByteArray { } } + assert(i == result.size) { "Should have reached the end" } + return result }