diff --git a/libraries/stdlib/jvm/src/kotlin/io/Console.kt b/libraries/stdlib/jvm/src/kotlin/io/Console.kt index 3b06f39dfea..1128adeffd6 100644 --- a/libraries/stdlib/jvm/src/kotlin/io/Console.kt +++ b/libraries/stdlib/jvm/src/kotlin/io/Console.kt @@ -8,11 +8,11 @@ package kotlin.io import java.io.InputStream -import java.nio.Buffer import java.nio.ByteBuffer import java.nio.CharBuffer import java.nio.charset.Charset import java.nio.charset.CharsetDecoder +import java.nio.charset.CoderResult /** Prints the given [message] to the standard output stream. */ @kotlin.internal.InlineOnly @@ -140,88 +140,137 @@ public actual inline fun println() { System.out.println() } -private const val BUFFER_SIZE: Int = 32 -private const val LINE_SEPARATOR_MAX_LENGTH: Int = 2 - -private val decoder: CharsetDecoder by lazy { Charset.defaultCharset().newDecoder() } - /** * Reads a line of input from the standard input stream. * * @return the line read or `null` if the input stream is redirected to a file and the end of file has been reached. */ -fun readLine(): String? = readLine(System.`in`, decoder) +fun readLine(): String? = LineReader.readLine(System.`in`, Charset.defaultCharset()) -internal fun readLine(inputStream: InputStream, decoder: CharsetDecoder): String? { - require(decoder.maxCharsPerByte() <= 1) { "Encodings with multiple chars per byte are not supported" } +// Singleton object lazy initializes on the first use, internal for tests +internal object LineReader { + private const val BUFFER_SIZE: Int = 32 + private lateinit var decoder: CharsetDecoder + private var directEOL = false + private val bytes = ByteArray(BUFFER_SIZE) + private val chars = CharArray(BUFFER_SIZE) + private val byteBuf: ByteBuffer = ByteBuffer.wrap(bytes) + private val charBuf: CharBuffer = CharBuffer.wrap(chars) + private val sb = StringBuilder() - val byteBuffer = ByteBuffer.allocate(BUFFER_SIZE) - val charBuffer = CharBuffer.allocate(LINE_SEPARATOR_MAX_LENGTH * 2) // twice for surrogate pairs - val stringBuilder = StringBuilder() - - var read = inputStream.read() - if (read == -1) return null - do { - byteBuffer.put(read.toByte()) - if (decoder.tryDecode(byteBuffer, charBuffer, false)) { - if (charBuffer.endsWithLineSeparator()) { - break + /** + * Reads line from the specified [inputStream] with the given [charset]. + * The general design: + * * This function contains only fast path code and all it state is kept in local variables as much as possible. + * * All the slow-path code is moved to separate functions and the call-sequence bytecode is minimized for it. + */ + @Synchronized + fun readLine(inputStream: InputStream, charset: Charset): String? { // charset == null -> use default + if (!::decoder.isInitialized || decoder.charset() != charset) updateCharset(charset) + var nBytes = 0 + var nChars = 0 + while (true) { + val readByte = inputStream.read() + if (readByte == -1) { + // The result is null only if there was absolutely nothing read + if (sb.isEmpty() && nBytes == 0 && nChars == 0) { + return null + } else { + nChars = decodeEndOfInput(nBytes, nChars) // throws exception if partial char + break + } + } else { + bytes[nBytes++] = readByte.toByte() } - if (charBuffer.remaining() < 2) { - charBuffer.offloadPrefixTo(stringBuilder) + // With "directEOL" encoding bytes are batched before being decoded all at once + if (readByte == '\n'.toInt() || nBytes == BUFFER_SIZE || !directEOL) { + // Decode the bytes that were read + byteBuf.limit(nBytes) // byteBuf position is always zero + charBuf.position(nChars) // charBuf limit is always BUFFER_SIZE + nChars = decode(false) + // Break when we have decoded end of line + if (nChars > 0 && chars[nChars - 1] == '\n') { + byteBuf.position(0) // reset position for next use + break + } + // otherwise we're going to read more bytes, so compact byteBuf + nBytes = compactBytes() } } - read = inputStream.read() - } while (read != -1) - - with(decoder) { - tryDecode(byteBuffer, charBuffer, true) // throws exception if undecoded bytes are left - reset() + // Trim the end of line + if (nChars > 0 && chars[nChars - 1] == '\n') { + nChars-- + if (nChars > 0 && chars[nChars - 1] == '\r') nChars-- + } + // Fast path for short lines (don't use StringBuilder) + if (sb.isEmpty()) return String(chars, 0, nChars) + // Copy the rest of chars to StringBuilder + sb.append(chars, 0, nChars) + // Build the result + val result = sb.toString() + if (sb.length > BUFFER_SIZE) trimStringBuilder() + sb.setLength(0) + return result } - with(charBuffer) { - var length = position() - if (length > 0 && get(length - 1) == '\n') { - length-- - if (length > 0 && get(length - 1) == '\r') { - length-- + // The result is the number of chars in charBuf + private fun decode(endOfInput: Boolean): Int { + while (true) { + val coderResult: CoderResult = decoder.decode(byteBuf, charBuf, endOfInput) + if (coderResult.isError) { + resetAll() // so that next call to readLine starts from clean state + coderResult.throwException() } - } - flip() - repeat(length) { - stringBuilder.append(get()) + val nChars = charBuf.position() + if (!coderResult.isOverflow) return nChars // has room in buffer -- everything possible was decoded + // overflow (charBuf is full) -- offload everything from charBuf but last char into sb + sb.append(chars, 0, nChars - 1) + charBuf.position(0) + charBuf.limit(BUFFER_SIZE) + charBuf.put(chars[nChars - 1]) // retain last char } } - return stringBuilder.toString() -} - -private fun CharsetDecoder.tryDecode(byteBuffer: ByteBuffer, charBuffer: CharBuffer, isEndOfStream: Boolean): Boolean { - val positionBefore = charBuffer.position() - byteBuffer.flip() - with(decode(byteBuffer, charBuffer, isEndOfStream)) { - if (isError) throwException() + // Slow path -- only on long lines (extra call to decode will be performed) + private fun compactBytes(): Int = with(byteBuf) { + compact() + return position().also { position(0) } } - return (charBuffer.position() > positionBefore).also { isDecoded -> - if (isDecoded) byteBuffer.clear() else byteBuffer.flipBack() + + // Slow path -- only on end of input + private fun decodeEndOfInput(nBytes: Int, nChars: Int): Int { + byteBuf.limit(nBytes) // byteBuf position is always zero + charBuf.position(nChars) // charBuf limit is always BUFFER_SIZE + return decode(true).also { // throws exception if partial char + // reset decoder and byteBuf for next use + decoder.reset() + byteBuf.position(0) + } + } + + // Slow path -- only on charset change + private fun updateCharset(charset: Charset) { + decoder = charset.newDecoder() + // try decoding ASCII line separator to see if this charset (like UTF-8) encodes it directly + byteBuf.clear() + charBuf.clear() + byteBuf.put('\n'.toByte()) + byteBuf.flip() + decoder.decode(byteBuf, charBuf, false) + directEOL = charBuf.position() == 1 && charBuf.get(0) == '\n' + resetAll() + } + + // Slow path -- only on exception in decoder and on charset change + private fun resetAll() { + decoder.reset() + byteBuf.position(0) + sb.setLength(0) + } + + // Slow path -- only on long lines + private fun trimStringBuilder() { + sb.setLength(BUFFER_SIZE) + sb.trimToSize() } } - -private fun CharBuffer.endsWithLineSeparator(): Boolean { - val p = position() - return p > 0 && get(p - 1) == '\n' -} - -private fun Buffer.flipBack() { - position(limit()) - limit(capacity()) -} - -/** Extracts everything except the last char into [builder]. */ -private fun CharBuffer.offloadPrefixTo(builder: StringBuilder) { - flip() - repeat(limit() - 1) { - builder.append(get()) - } - compact() -} diff --git a/libraries/stdlib/jvm/test/io/Console.kt b/libraries/stdlib/jvm/test/io/Console.kt index f3e03873069..7dd350ea589 100644 --- a/libraries/stdlib/jvm/test/io/Console.kt +++ b/libraries/stdlib/jvm/test/io/Console.kt @@ -9,6 +9,8 @@ package test.io import org.junit.Test import java.nio.charset.Charset +import kotlin.random.Random +import kotlin.random.nextInt import kotlin.test.* class ConsoleTest { @@ -64,6 +66,74 @@ class ConsoleTest { testReadLine("first${linuxLineSeparator}second", listOf("first", "second"), charset = Charsets.UTF_32) } + @Test + fun shouldReadAllSupportedEncodings() { + val lines = listOf( + "ONE", "TWICE", "", "0123456", + "This is a very long line that will overflow buffers that are allocated in the code of LineReader object", + "This line is quite short", + "x".repeat(1000), // stress + "7", "8", "9" // some short stuff at the end + ) + // Filter all available charsets that can be encoded + val charsets: List = Charset.availableCharsets().values.filter { charset -> + try { + charset.newEncoder() + true // take it + } catch (e: UnsupportedOperationException) { + false // we can only test charset that supports encoding, skip it + } + } + // Run the test + for (separator in listOf(linuxLineSeparator, windowsLineSeparator)) { + val text = lines.joinToString(separator) + for (charset in charsets) { + val reference = readLinesReference(text, charset) + if (reference != lines) continue // this encoding does not support ASCII chars that we test, skip + // Now we can test readLine function + val actual = readLines(text, charset) + assertEquals(lines, actual, "Comparing with $charset") + } + } + } + + @Test + fun shouldReadAllUnicodeCodePoints() { + // Generate lines of ever-increasing length with sample unicode code points to stress all corner-cases in + // line lengths and ability to handle different bit-lengths of unicode code points. + var cp = 0 + val rnd = Random(1) + val logFactor = 7 // log of number of code points that are sampled per each bit length of code point + fun nextCP(): Int { + if (cp == 10 || cp == 13) cp++ // skip line endings + if (cp in 0xD800..0xFFFF) cp = 0x10000 // skip surrogates + // to make the test run faster don't test all code points, the larger they are, the sparser they are sampled + // For each bit length of the code point we randomly sample ~2^logFactor code points for this test + val maxStep = cp.coerceAtLeast(1 shl logFactor).takeHighestOneBit() shr logFactor + val step = rnd.nextInt(1..maxStep) + return cp.also { cp += step } + } + val lines = ArrayList().apply { + var len = 1 + while (cp < Character.MAX_CODE_POINT) { + add(buildString { + repeat(len) { + appendCodePoint(nextCP()) + if (cp >= Character.MAX_CODE_POINT) return@buildString + } + }) + len++ + } + } + // test all standard unicode encoding that should be able to represent all code points + for (separator in listOf(linuxLineSeparator, windowsLineSeparator)) { + val text = lines.joinToString(separator) + for (charset in listOf(Charsets.UTF_8, Charsets.UTF_16BE, Charsets.UTF_16LE, Charsets.UTF_32BE, Charsets.UTF_32LE)) { + testReadLine(text, lines, charset) + } + } + } + @Test fun readSurrogatePairs() { val c = "\uD83D\uDC4D" // thumb-up emoji @@ -76,17 +146,15 @@ class ConsoleTest { private fun testReadLine(text: String, expected: List, charset: Charset = Charsets.UTF_8) { val actual = readLines(text, charset) - assertEquals(expected, actual) + assertEquals(expected, actual, "Comparing with $charset") val referenceExpected = readLinesReference(text, charset) assertEquals(referenceExpected, actual, "Comparing to reference readLine") - } private fun readLines(text: String, charset: Charset): List { text.byteInputStream(charset).use { stream -> - val decoder = charset.newDecoder() @Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") - return generateSequence { readLine(stream, decoder) }.toList().also { + return generateSequence { LineReader.readLine(stream, charset) }.toList().also { assertTrue("All bytes should be read") { stream.read() == -1 } } }