Avoid allocating large buffers in File.writeText #KT-51058

Also fixes File.appendText and Path.write/appendText.


Merge-request: KT-MR-12804
Merged-by: Abduqodiri Qurbonzoda <abduqodiri.qurbonzoda@jetbrains.com>

libraries/stdlib/jvm/src/kotlin/io/FileReadWrite.kt

@@ -9,9 +9,13 @@
 package kotlin.io
 
 import java.io.*
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
 import java.util.*
 import java.nio.charset.Charset
-import kotlin.internal.*
+import java.nio.charset.CharsetEncoder
+import java.nio.charset.CodingErrorAction
+import kotlin.math.ceil
 
 
 /**
@@ -131,7 +135,8 @@ public fun File.readText(charset: Charset = Charsets.UTF_8): String = reader(cha
  * @param text text to write into file.
  * @param charset character set to use.
  */
-public fun File.writeText(text: String, charset: Charset = Charsets.UTF_8): Unit = writeBytes(text.toByteArray(charset))
+public fun File.writeText(text: String, charset: Charset = Charsets.UTF_8): Unit =
+    FileOutputStream(this).use { it.writeTextImpl(text, charset) }
 
 /**
  * Appends [text] to the content of this file using UTF-8 or the specified [charset].
@@ -139,7 +144,54 @@ public fun File.writeText(text: String, charset: Charset = Charsets.UTF_8): Unit
  * @param text text to append to file.
  * @param charset character set to use.
  */
-public fun File.appendText(text: String, charset: Charset = Charsets.UTF_8): Unit = appendBytes(text.toByteArray(charset))
+public fun File.appendText(text: String, charset: Charset = Charsets.UTF_8): Unit =
+    FileOutputStream(this, true).use { it.writeTextImpl(text, charset) }
+
+internal fun OutputStream.writeTextImpl(text: String, charset: Charset) {
+    val chunkSize = DEFAULT_BUFFER_SIZE
+
+    if (text.length < 2 * chunkSize) {
+        this.write(text.toByteArray(charset))
+        return
+    }
+
+    val encoder = charset.newReplaceEncoder()
+    val charBuffer = CharBuffer.allocate(chunkSize)
+    val byteBuffer = byteBufferForEncoding(chunkSize, encoder)
+
+    var startIndex = 0
+    var leftover = 0
+
+    while (startIndex < text.length) {
+        val copyLength = minOf(chunkSize - leftover, text.length - startIndex)
+        val endIndex = startIndex + copyLength
+
+        text.toCharArray(charBuffer.array(), leftover, startIndex, endIndex)
+        charBuffer.limit(copyLength + leftover)
+        encoder.encode(charBuffer, byteBuffer, /*endOfInput = */endIndex == text.length).also { check(it.isUnderflow) }
+        this.write(byteBuffer.array(), 0, byteBuffer.position())
+
+        if (charBuffer.position() != charBuffer.limit()) {
+            charBuffer.put(0, charBuffer.get()) // the last char is a high surrogate
+            leftover = 1
+        } else {
+            leftover = 0
+        }
+
+        charBuffer.clear()
+        byteBuffer.clear()
+        startIndex = endIndex
+    }
+}
+
+internal fun Charset.newReplaceEncoder() = newEncoder()
+    .onMalformedInput(CodingErrorAction.REPLACE)
+    .onUnmappableCharacter(CodingErrorAction.REPLACE)
+
+internal fun byteBufferForEncoding(chunkSize: Int, encoder: CharsetEncoder): ByteBuffer {
+    val maxBytesPerChar = ceil(encoder.maxBytesPerChar()).toInt() // including replacement sequence
+    return ByteBuffer.allocate(chunkSize * maxBytesPerChar)
+}
 
 /**
  * Reads file by byte blocks and calls [action] for each block read.