azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[KLIB] Change KLIB IO API

Browse Source 
- Use ReadBuffer instead of native java buffer
 - Use Weak buffers for File IO
 - fix klib memory leak for KT-40345
This commit is contained in:
Roman Artemev
2020-07-31 13:58:50 +03:00
committed by romanart
parent 0e54f98b79
commit add0ad6733
2 changed files with 122 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt
+30 -24
View File
@@ -7,9 +7,8 @@ package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.konan.file.File
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
abstract class IrArrayReader(private val buffer: ByteBuffer) {
abstract class IrArrayReader(private val buffer: ReadBuffer) {
private val indexToOffset: IntArray
fun entryCount() = indexToOffset.size - 1
@@ -28,21 +27,21 @@ abstract class IrArrayReader(private val buffer: ByteBuffer) {
val offset = indexToOffset[id]
val size = indexToOffset[id + 1] - offset
val result = ByteArray(size)
buffer.position(offset)
buffer.position = offset
buffer.get(result, 0, size)
return result
}
}
class IrArrayFileReader(file: File) : IrArrayReader(file.readBuffer)
class IrArrayMemoryReader(bytes: ByteArray) : IrArrayReader(bytes.buffer)
class IrArrayFileReader(file: File) : IrArrayReader(ReadBuffer.WeakFileBuffer(file.javaFile()))
class IrArrayMemoryReader(bytes: ByteArray) : IrArrayReader(ReadBuffer.MemoryBuffer(bytes))
abstract class IrMultiArrayReader(private val buffer: ByteBuffer) {
abstract class IrMultiArrayReader(private val buffer: ReadBuffer) {
private val indexToOffset: IntArray
private val indexIndexToOffset = mutableMapOf<Int, IntArray>()
private fun readOffsets(position: Int): IntArray {
buffer.position(position)
buffer.position = position
val count = buffer.int
val result = IntArray(count + 1)
result[0] = 4 * (count + 1)
@@ -62,7 +61,7 @@ abstract class IrMultiArrayReader(private val buffer: ByteBuffer) {
val offset = indexToOffset[id]
val size = indexToOffset[id + 1] - offset
val result = ByteArray(size)
buffer.position(offset)
buffer.position = offset
buffer.get(result, 0, size)
return result
}
@@ -78,22 +77,22 @@ abstract class IrMultiArrayReader(private val buffer: ByteBuffer) {
val dataSize = collumnOffsets[column + 1] - dataOffset
val result = ByteArray(dataSize)
buffer.position(rowOffset + dataOffset)
buffer.position = rowOffset + dataOffset
buffer.get(result, 0, dataSize)
return result
}
}
class IrMultiArrayFileReader(file: File) : IrMultiArrayReader(file.readBuffer)
class IrMultiArrayMemoryReader(bytes: ByteArray) : IrMultiArrayReader(bytes.buffer)
class IrMultiArrayFileReader(file: File) : IrMultiArrayReader(ReadBuffer.WeakFileBuffer(file.javaFile()))
class IrMultiArrayMemoryReader(bytes: ByteArray) : IrMultiArrayReader(ReadBuffer.MemoryBuffer(bytes))
abstract class IrMultiTableReader<K>(private val buffer: ByteBuffer, private val keyReader: ByteBuffer.() -> K) {
abstract class IrMultiTableReader<K>(private val buffer: ReadBuffer, private val keyReader: ReadBuffer.() -> K) {
private val indexToOffset: IntArray
private val indexToIndexMap = mutableMapOf<Int, Map<K, Pair<Int, Int>>>()
private fun readOffsets(position: Int): IntArray {
buffer.position(position)
buffer.position = position
val count = buffer.int
val result = IntArray(count + 1)
result[0] = 4 * (count + 1)
@@ -110,7 +109,7 @@ abstract class IrMultiTableReader<K>(private val buffer: ByteBuffer, private val
}
private fun readIndexMap(position: Int): Map<K, Pair<Int, Int>> {
buffer.position(position)
buffer.position = position
val result = mutableMapOf<K, Pair<Int, Int>>()
val count = buffer.int
@@ -138,13 +137,13 @@ abstract class IrMultiTableReader<K>(private val buffer: ByteBuffer, private val
val offset = coordinates.first
val size = coordinates.second
val result = ByteArray(size)
buffer.position(rowOffset + offset)
buffer.position = rowOffset + offset
buffer.get(result, 0, size)
return result
}
}
abstract class IrTableReader<K>(private val buffer: ByteBuffer, keyReader: ByteBuffer.() -> K) {
abstract class IrTableReader<K>(private val buffer: ReadBuffer, keyReader: ReadBuffer.() -> K) {
private val indexToOffset = mutableMapOf<K, Pair<Int, Int>>()
init {
@@ -163,22 +162,29 @@ abstract class IrTableReader<K>(private val buffer: ByteBuffer, keyReader: ByteB
val offset = coordinates.first
val size = coordinates.second
val result = ByteArray(size)
buffer.position(offset)
buffer.position = offset
buffer.get(result, 0, size)
return result
}
}
val File.readBuffer: ByteBuffer get() = this.readBytes().buffer
val ByteArray.buffer: ByteBuffer get() = ByteBuffer.wrap(this)
class IndexIrTableFileReader(file: File) : IrTableReader<Long>(file.readBuffer, { long })
class IndexIrTableMemoryReader(bytes: ByteArray) : IrTableReader<Long>(bytes.buffer, { long })
fun File.javaFile(): java.io.File = java.io.File(path)
class IndexIrTableFileReader(file: File) : IrTableReader<Long>(ReadBuffer.WeakFileBuffer(file.javaFile()), { long })
class IndexIrTableMemoryReader(bytes: ByteArray) : IrTableReader<Long>(ReadBuffer.MemoryBuffer(bytes), { long })
data class DeclarationId(val id: Int)
class DeclarationIrTableFileReader(file: File) : IrTableReader<DeclarationId>(file.readBuffer, { DeclarationId(int) })
class DeclarationIrTableMemoryReader(bytes: ByteArray) : IrTableReader<DeclarationId>(bytes.buffer, { DeclarationId(int) })
class DeclarationIrTableFileReader(file: File) :
IrTableReader<DeclarationId>(ReadBuffer.WeakFileBuffer(file.javaFile()), { DeclarationId(int) })
class DeclarationIrMultiTableFileReader(file: File) : IrMultiTableReader<DeclarationId>(file.readBuffer, { DeclarationId(int) })
class DeclarationIrMultiTableMemoryReader(bytes: ByteArray) : IrMultiTableReader<DeclarationId>(bytes.buffer, { DeclarationId(int) })
class DeclarationIrTableMemoryReader(bytes: ByteArray) :
IrTableReader<DeclarationId>(ReadBuffer.MemoryBuffer(bytes), { DeclarationId(int) })
class DeclarationIrMultiTableFileReader(file: File) :
IrMultiTableReader<DeclarationId>(ReadBuffer.WeakFileBuffer(file.javaFile()), { DeclarationId(int) })
class DeclarationIrMultiTableMemoryReader(bytes: ByteArray) :
IrMultiTableReader<DeclarationId>(ReadBuffer.MemoryBuffer(bytes), { DeclarationId(int) })
compiler/util-klib/src/org/jetbrains/kotlin/library/impl/ReadBuffer.kt
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.library.impl
import java.io.File
import java.lang.ref.SoftReference
import java.nio.ByteBuffer
sealed class ReadBuffer {
abstract val size: Int
abstract fun get(result: ByteArray, offset: Int, length: Int)
abstract var position: Int
abstract val int: Int
abstract val long: Long
abstract class NIOReader(private val buffer: ByteBuffer) : ReadBuffer() {
override val size: Int
get() = buffer.limit()
override fun get(result: ByteArray, offset: Int, length: Int) {
buffer.get(result, offset, length)
}
override var position: Int
get() = buffer.position()
set(value) { buffer.position(value) }
override val int: Int
get() = buffer.int
override val long: Long
get() = buffer.long
}
class MemoryBuffer(bytes: ByteArray) : NIOReader(bytes.buffer)
class DirectFileBuffer(file: File) : NIOReader(file.readBytes().buffer)
class WeakFileBuffer(private val file: File) : ReadBuffer() {
override val size: Int
get() = file.length().toInt()
override fun get(result: ByteArray, offset: Int, length: Int) {
val buf = ensureBuffer()
pos += length
buf.get(result, offset, length)
}
override val int: Int
get(): Int {
val buf = ensureBuffer()
pos += Int.SIZE_BYTES
return buf.int
}
override val long: Long
get(): Long {
val buf = ensureBuffer()
pos += Long.SIZE_BYTES
return buf.long
}
private var pos: Int = 0
override var position: Int
get() = pos.also { assert(it == ensureBuffer().position()) }
set(value) {
val buf = ensureBuffer()
pos = value
buf.position(value)
}
private fun ensureBuffer(): ByteBuffer {
var tmpBuffer = weakBuffer.get()
if (tmpBuffer == null) {
tmpBuffer = file.readBytes().buffer
tmpBuffer.position(pos)
weakBuffer = SoftReference(tmpBuffer)
}
return tmpBuffer
}
private var weakBuffer: SoftReference<ByteBuffer> = SoftReference(null)
}
}