From add0ad67335b2f58f4c941d6747f8eb4b57a307c Mon Sep 17 00:00:00 2001 From: Roman Artemev Date: Fri, 31 Jul 2020 13:58:50 +0300 Subject: [PATCH] [KLIB] Change KLIB IO API - Use ReadBuffer instead of native java buffer - Use Weak buffers for File IO - fix klib memory leak for KT-40345 --- .../kotlin/library/impl/IrFileReaders.kt | 54 ++++++----- .../kotlin/library/impl/ReadBuffer.kt | 92 +++++++++++++++++++ 2 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 compiler/util-klib/src/org/jetbrains/kotlin/library/impl/ReadBuffer.kt diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt index c6e8bfdf5d8..ec478c16a95 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt @@ -7,9 +7,8 @@ package org.jetbrains.kotlin.library.impl import org.jetbrains.kotlin.konan.file.File import java.nio.ByteBuffer -import java.nio.channels.FileChannel -abstract class IrArrayReader(private val buffer: ByteBuffer) { +abstract class IrArrayReader(private val buffer: ReadBuffer) { private val indexToOffset: IntArray fun entryCount() = indexToOffset.size - 1 @@ -28,21 +27,21 @@ abstract class IrArrayReader(private val buffer: ByteBuffer) { val offset = indexToOffset[id] val size = indexToOffset[id + 1] - offset val result = ByteArray(size) - buffer.position(offset) + buffer.position = offset buffer.get(result, 0, size) return result } } -class IrArrayFileReader(file: File) : IrArrayReader(file.readBuffer) -class IrArrayMemoryReader(bytes: ByteArray) : IrArrayReader(bytes.buffer) +class IrArrayFileReader(file: File) : IrArrayReader(ReadBuffer.WeakFileBuffer(file.javaFile())) +class IrArrayMemoryReader(bytes: ByteArray) : IrArrayReader(ReadBuffer.MemoryBuffer(bytes)) -abstract class IrMultiArrayReader(private val buffer: ByteBuffer) { +abstract class IrMultiArrayReader(private val buffer: ReadBuffer) { private val indexToOffset: IntArray private val indexIndexToOffset = mutableMapOf() private fun readOffsets(position: Int): IntArray { - buffer.position(position) + buffer.position = position val count = buffer.int val result = IntArray(count + 1) result[0] = 4 * (count + 1) @@ -62,7 +61,7 @@ abstract class IrMultiArrayReader(private val buffer: ByteBuffer) { val offset = indexToOffset[id] val size = indexToOffset[id + 1] - offset val result = ByteArray(size) - buffer.position(offset) + buffer.position = offset buffer.get(result, 0, size) return result } @@ -78,22 +77,22 @@ abstract class IrMultiArrayReader(private val buffer: ByteBuffer) { val dataSize = collumnOffsets[column + 1] - dataOffset val result = ByteArray(dataSize) - buffer.position(rowOffset + dataOffset) + buffer.position = rowOffset + dataOffset buffer.get(result, 0, dataSize) return result } } -class IrMultiArrayFileReader(file: File) : IrMultiArrayReader(file.readBuffer) -class IrMultiArrayMemoryReader(bytes: ByteArray) : IrMultiArrayReader(bytes.buffer) +class IrMultiArrayFileReader(file: File) : IrMultiArrayReader(ReadBuffer.WeakFileBuffer(file.javaFile())) +class IrMultiArrayMemoryReader(bytes: ByteArray) : IrMultiArrayReader(ReadBuffer.MemoryBuffer(bytes)) -abstract class IrMultiTableReader(private val buffer: ByteBuffer, private val keyReader: ByteBuffer.() -> K) { +abstract class IrMultiTableReader(private val buffer: ReadBuffer, private val keyReader: ReadBuffer.() -> K) { private val indexToOffset: IntArray private val indexToIndexMap = mutableMapOf>>() private fun readOffsets(position: Int): IntArray { - buffer.position(position) + buffer.position = position val count = buffer.int val result = IntArray(count + 1) result[0] = 4 * (count + 1) @@ -110,7 +109,7 @@ abstract class IrMultiTableReader(private val buffer: ByteBuffer, private val } private fun readIndexMap(position: Int): Map> { - buffer.position(position) + buffer.position = position val result = mutableMapOf>() val count = buffer.int @@ -138,13 +137,13 @@ abstract class IrMultiTableReader(private val buffer: ByteBuffer, private val val offset = coordinates.first val size = coordinates.second val result = ByteArray(size) - buffer.position(rowOffset + offset) + buffer.position = rowOffset + offset buffer.get(result, 0, size) return result } } -abstract class IrTableReader(private val buffer: ByteBuffer, keyReader: ByteBuffer.() -> K) { +abstract class IrTableReader(private val buffer: ReadBuffer, keyReader: ReadBuffer.() -> K) { private val indexToOffset = mutableMapOf>() init { @@ -163,22 +162,29 @@ abstract class IrTableReader(private val buffer: ByteBuffer, keyReader: ByteB val offset = coordinates.first val size = coordinates.second val result = ByteArray(size) - buffer.position(offset) + buffer.position = offset buffer.get(result, 0, size) return result } } -val File.readBuffer: ByteBuffer get() = this.readBytes().buffer val ByteArray.buffer: ByteBuffer get() = ByteBuffer.wrap(this) -class IndexIrTableFileReader(file: File) : IrTableReader(file.readBuffer, { long }) -class IndexIrTableMemoryReader(bytes: ByteArray) : IrTableReader(bytes.buffer, { long }) +fun File.javaFile(): java.io.File = java.io.File(path) + +class IndexIrTableFileReader(file: File) : IrTableReader(ReadBuffer.WeakFileBuffer(file.javaFile()), { long }) +class IndexIrTableMemoryReader(bytes: ByteArray) : IrTableReader(ReadBuffer.MemoryBuffer(bytes), { long }) data class DeclarationId(val id: Int) -class DeclarationIrTableFileReader(file: File) : IrTableReader(file.readBuffer, { DeclarationId(int) }) -class DeclarationIrTableMemoryReader(bytes: ByteArray) : IrTableReader(bytes.buffer, { DeclarationId(int) }) +class DeclarationIrTableFileReader(file: File) : + IrTableReader(ReadBuffer.WeakFileBuffer(file.javaFile()), { DeclarationId(int) }) -class DeclarationIrMultiTableFileReader(file: File) : IrMultiTableReader(file.readBuffer, { DeclarationId(int) }) -class DeclarationIrMultiTableMemoryReader(bytes: ByteArray) : IrMultiTableReader(bytes.buffer, { DeclarationId(int) }) \ No newline at end of file +class DeclarationIrTableMemoryReader(bytes: ByteArray) : + IrTableReader(ReadBuffer.MemoryBuffer(bytes), { DeclarationId(int) }) + +class DeclarationIrMultiTableFileReader(file: File) : + IrMultiTableReader(ReadBuffer.WeakFileBuffer(file.javaFile()), { DeclarationId(int) }) + +class DeclarationIrMultiTableMemoryReader(bytes: ByteArray) : + IrMultiTableReader(ReadBuffer.MemoryBuffer(bytes), { DeclarationId(int) }) diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/ReadBuffer.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/ReadBuffer.kt new file mode 100644 index 00000000000..8130df8355a --- /dev/null +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/ReadBuffer.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.library.impl + +import java.io.File +import java.lang.ref.SoftReference +import java.nio.ByteBuffer + +sealed class ReadBuffer { + + abstract val size: Int + abstract fun get(result: ByteArray, offset: Int, length: Int) + abstract var position: Int + + + abstract val int: Int + abstract val long: Long + + abstract class NIOReader(private val buffer: ByteBuffer) : ReadBuffer() { + + override val size: Int + get() = buffer.limit() + + override fun get(result: ByteArray, offset: Int, length: Int) { + buffer.get(result, offset, length) + } + + override var position: Int + get() = buffer.position() + set(value) { buffer.position(value) } + + override val int: Int + get() = buffer.int + + override val long: Long + get() = buffer.long + } + + class MemoryBuffer(bytes: ByteArray) : NIOReader(bytes.buffer) + + class DirectFileBuffer(file: File) : NIOReader(file.readBytes().buffer) + + class WeakFileBuffer(private val file: File) : ReadBuffer() { + override val size: Int + get() = file.length().toInt() + + override fun get(result: ByteArray, offset: Int, length: Int) { + val buf = ensureBuffer() + pos += length + buf.get(result, offset, length) + } + + override val int: Int + get(): Int { + val buf = ensureBuffer() + pos += Int.SIZE_BYTES + return buf.int + } + + override val long: Long + get(): Long { + val buf = ensureBuffer() + pos += Long.SIZE_BYTES + return buf.long + } + + private var pos: Int = 0 + + override var position: Int + get() = pos.also { assert(it == ensureBuffer().position()) } + set(value) { + val buf = ensureBuffer() + pos = value + buf.position(value) + } + + private fun ensureBuffer(): ByteBuffer { + var tmpBuffer = weakBuffer.get() + if (tmpBuffer == null) { + tmpBuffer = file.readBytes().buffer + tmpBuffer.position(pos) + weakBuffer = SoftReference(tmpBuffer) + } + return tmpBuffer + } + + private var weakBuffer: SoftReference = SoftReference(null) + } +} \ No newline at end of file