azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Combine all IR files to a single mmaped file with an index. (#2620)

Browse Source
This commit is contained in:
Nikolay Igotti
2019-02-04 18:30:38 +03:00
committed by GitHub
parent 67d6d47e5f
commit c1fa7f0f79
13 changed files with 181 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt
+18 -4
View File
@@ -1,3 +1,19 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
@@ -46,10 +62,8 @@ interface KonanLibraryLayout {
File(packageFragmentsDir(packageFqName), "$partName$KLIB_METADATA_FILE_EXTENSION_WITH_DOT")
val irDir
get() = File(libDir, "ir")
fun hiddenDeclarationFile(declarationId: String)
= File(irDir, "hidden_$declarationId.knd")
fun visibleDeclarationFile(declarationId: String)
= File(irDir, "visible_$declarationId.knd")
val irFile
get() = File(irDir, "irCombined.knd")
val irHeader
get() = File(irDir, "irHeaders.kni")
val irIndex: File
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt
-2
View File
@@ -3,6 +3,4 @@ package org.jetbrains.kotlin.konan.library
interface MetadataReader {
fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray
fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray
fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray
}
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt
+94
View File
@@ -0,0 +1,94 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import java.io.RandomAccessFile
import java.nio.channels.FileChannel
data class DeclarationId(val id: Long, val isLocal: Boolean)
class CombinedIrFileReader(file: File) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
init {
val declarationsCount = buffer.int
for (i in 0 until declarationsCount) {
val id = buffer.long
val isLocal = buffer.int != 0
val offset = buffer.int
val size = buffer.int
declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
}
}
fun declarationBytes(id: DeclarationId): ByteArray {
val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
val result = ByteArray(offsetSize.second)
buffer.position(offsetSize.first)
buffer.get(result, 0, offsetSize.second)
return result
}
}
private const val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int).
private const val INDEX_HEADER_SIZE = 4 // sizeof(Int).
class CombinedIrFileWriter(val declarationCount: Int) {
private var currentDeclaration = 0
private var currentPosition = 0
private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
private val randomAccessFile = RandomAccessFile(file.path, "rw")
init {
randomAccessFile.writeInt(declarationCount)
assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
for (i in 0 until declarationCount) {
randomAccessFile.writeLong(-1) // id
randomAccessFile.writeInt(-1) // isLocal
randomAccessFile.writeInt(-1) // offset
randomAccessFile.writeInt(-1) // size
}
currentPosition = randomAccessFile.filePointer.toInt()
assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
}
fun skipDeclaration() {
currentDeclaration++
}
fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
randomAccessFile.writeLong(id.id)
randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
randomAccessFile.writeInt(currentPosition)
randomAccessFile.writeInt(bytes.size)
randomAccessFile.seek(currentPosition.toLong())
randomAccessFile.write(bytes)
assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
currentPosition = randomAccessFile.filePointer.toInt()
currentDeclaration++
}
fun finishWriting(): File {
assert(currentDeclaration == declarationCount)
randomAccessFile.close()
return file
}
}
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt
-11
View File
@@ -11,15 +11,4 @@ internal object DefaultMetadataReaderImpl : MetadataReader {
override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray =
libraryLayout.packageFragmentFile(fqName, partName).readBytes()
override fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray =
libraryLayout.irHeader.readBytes()
override fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray {
val name = index.toULong().toString(16)
val file = if (isLocal)
libraryLayout.hiddenDeclarationFile(name)
else
libraryLayout.visibleDeclarationFile(name)
return file.readBytes()
}
}
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt
+15 -2
View File
@@ -84,9 +84,22 @@ class KonanLibraryImpl(
}
}
override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let { if (it.exists) metadataReader.loadIrHeader(library) else null }}}
override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let {
if (it.exists) loadIrHeader() else null }
}}
override fun irDeclaration(index: Long, isLocal: Boolean) = layout.inPlace { metadataReader.loadIrDeclaraton(it, index, isLocal) }
override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal)
override fun toString() = "$libraryName[default=$isDefault]"
private val combinedDeclarations: CombinedIrFileReader by lazy {
CombinedIrFileReader(layout.realFiles {
it.irFile
})
}
private fun loadIrHeader(): ByteArray =
layout.inPlace { it.irHeader.readBytes() }
private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
combinedDeclarations.declarationBytes(DeclarationId(index, isLocal))
}
shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt
+3 -1
View File
@@ -19,7 +19,7 @@ private class ZippedKonanLibraryLayout(val klibFile: File, override val target:
override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
override val libDir: File= File("/")
override val libDir = File("/")
override fun <T> realFiles(action: (KonanLibraryLayout) -> T): T {
return action(FileExtractor(this))!!
@@ -70,6 +70,8 @@ private class FileExtractor(val zippedLibraryLayout: ZippedKonanLibraryLayout):
override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
override val irFile: File by lazy { extract(super.irFile) }
fun extract(file: File): File = zippedLibraryLayout.klibFile.withZipFileSystem { zipFileSystem ->
val temporary = createTempFile(file.name)
zipFileSystem.file(file).copyTo(temporary)
shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt
+17 -1
View File
@@ -1,3 +1,19 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.konan
fun String.parseKonanAbiVersion(): KonanAbiVersion {
@@ -6,7 +22,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion {
data class KonanAbiVersion(val version: Int) {
companion object {
val CURRENT = KonanAbiVersion(6)
val CURRENT = KonanAbiVersion(7)
}
override fun toString() = "$version"
}
shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt
+13 -1
View File
@@ -22,7 +22,10 @@ import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
import java.io.RandomAccessFile
import java.net.URI
import java.nio.MappedByteBuffer
import java.nio.channels.FileChannel
import java.nio.file.*
import java.nio.file.attribute.BasicFileAttributes
@@ -108,8 +111,17 @@ data class File constructor(internal val javaPath: Path) {
return FileVisitResult.CONTINUE
}
})
}
fun map(mode: FileChannel.MapMode = FileChannel.MapMode.READ_ONLY,
start: Long = 0, size: Long = -1): MappedByteBuffer {
val file = RandomAccessFile(path,
if (mode == FileChannel.MapMode.READ_ONLY) "r" else "rw")
val fileSize = if (mode == FileChannel.MapMode.READ_ONLY)
file.length() else size.also { assert(size != -1L) }
return file.channel.map(mode, start, fileSize) // Shall we .also { file.close() }?
}
fun deleteOnExit(): File {
// Works only on the default file system,
// but that's okay for now.