Moved the common klib reader functionality from Kotlin/Native to util-klib

This commit is contained in:
Alexander Gorshenev
2019-05-31 16:51:52 +03:00
committed by alexander-gorshenev
parent d1390365de
commit 7ddbd8ca80
10 changed files with 606 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
plugins {
kotlin("jvm")
id("jps-compatible")
}
description = "Common klib reader and writer"
dependencies {
compile(kotlinStdlib())
compile(project(":kotlin-util-io"))
}
sourceSets {
"main" { projectDefault() }
"test" { none() }
}
publish()
standardPublicJars()
@@ -0,0 +1,29 @@
package org.jetbrains.kotlin.library
import org.jetbrains.kotlin.konan.parseKonanVersion
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.KonanVersion
data class KonanLibraryVersioning(
val libraryVersion: String?,
val compilerVersion: KonanVersion?,
val abiVersion: KotlinAbiVersion?
)
fun Properties.writeKonanLibraryVersioning(versions: KonanLibraryVersioning) {
versions.abiVersion ?. let { this.setProperty(KLIB_PROPERTY_ABI_VERSION, it.toString()) }
versions.libraryVersion ?. let { this.setProperty(KLIB_PROPERTY_LIBRARY_VERSION, it) }
versions.compilerVersion ?. let { this.setProperty(KLIB_PROPERTY_COMPILER_VERSION, "${versions.compilerVersion.toString(true, true)}") }
}
fun Properties.readKonanLibraryVersioning(): KonanLibraryVersioning {
val abiVersion = this.getProperty(KLIB_PROPERTY_ABI_VERSION)?.parseKonanAbiVersion()
val libraryVersion = this.getProperty(KLIB_PROPERTY_LIBRARY_VERSION)
val compilerVersion = this.getProperty(KLIB_PROPERTY_COMPILER_VERSION)?.parseKonanVersion()
return KonanLibraryVersioning(
abiVersion = abiVersion,
libraryVersion = libraryVersion,
compilerVersion = compilerVersion
)
}
@@ -0,0 +1,28 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.library
fun String.parseKonanAbiVersion(): KotlinAbiVersion {
return KotlinAbiVersion(this.toInt())
}
data class KotlinAbiVersion(val version: Int) {
companion object {
val CURRENT = KotlinAbiVersion(11)
}
override fun toString() = "$version"
}
@@ -0,0 +1,51 @@
package org.jetbrains.kotlin.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.propertyList
const val KLIB_PROPERTY_ABI_VERSION = "abi_version"
const val KLIB_PROPERTY_COMPILER_VERSION = "compiler_version"
const val KLIB_PROPERTY_DEPENDENCY_VERSION = "dependency_version"
const val KLIB_PROPERTY_LIBRARY_VERSION = "library_version"
const val KLIB_PROPERTY_UNIQUE_NAME = "unique_name"
const val KLIB_PROPERTY_DEPENDS = "depends"
const val KLIB_PROPERTY_PACKAGE = "package"
/**
* Abstractions for getting access to the information stored inside of Kotlin/Native library.
*/
interface BaseKotlinLibrary {
val libraryName: String
val libraryFile: File
val versions: KonanLibraryVersioning
// Whether this library is default (provided by distribution)?
val isDefault: Boolean
val manifestProperties: Properties
}
interface MetadataLibrary {
val moduleHeaderData: ByteArray
fun packageMetadataParts(fqName: String): Set<String>
fun packageMetadata(fqName: String, partName: String): ByteArray
}
interface IrLibrary {
val dataFlowGraph: ByteArray?
val irHeader: ByteArray?
fun irDeclaration(index: Long, isLocal: Boolean): ByteArray
}
val BaseKotlinLibrary.uniqueName: String
get() = manifestProperties.getProperty(KLIB_PROPERTY_UNIQUE_NAME)!!
val BaseKotlinLibrary.unresolvedDependencies: List<UnresolvedLibrary>
get() = manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true)
.map {
UnresolvedLibrary(it, manifestProperties.getProperty("dependency_version_$it"))
}
interface BackendLibrary : BaseKotlinLibrary, MetadataLibrary, IrLibrary
// typealias JsLibrary = BackendLibrary?
@@ -0,0 +1,57 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.library.KLIB_METADATA_FILE_EXTENSION_WITH_DOT
/**
* This scheme describes the Kotlin/Native Library (KLIB) layout.
*/
interface KotlinLibraryLayout {
val libDir: File
val libraryName: String
get() = libDir.path
val manifestFile
get() = File(libDir, "manifest")
val resourcesDir
get() = File(libDir, "resources")
}
interface MetadataKotlinLibraryLayout : KotlinLibraryLayout {
val metadataDir
get() = File(libDir, "linkdata")
val moduleHeaderFile
get() = File(metadataDir, "module")
fun packageFragmentsDir(packageName: String) =
File(metadataDir, if (packageName == "") "root_package" else "package_$packageName")
fun packageFragmentFile(packageFqName: String, partName: String) =
File(packageFragmentsDir(packageFqName), "$partName$KLIB_METADATA_FILE_EXTENSION_WITH_DOT")
}
interface IrKotlinLibraryLayout : KotlinLibraryLayout {
val irDir
get() = File(libDir, "ir")
val irFile
get() = File(irDir, "irCombined.knd")
val irHeader
get() = File(irDir, "irHeaders.kni")
val dataFlowGraphFile
get() = File(irDir, "module_data_flow_graph")
}
@@ -0,0 +1,39 @@
package org.jetbrains.kotlin.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.file
import org.jetbrains.kotlin.konan.file.withMutableZipFileSystem
import org.jetbrains.kotlin.library.impl.zippedKotlinLibraryChecks
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
const val KLIB_METADATA_FILE_EXTENSION = "knm"
const val KLIB_METADATA_FILE_EXTENSION_WITH_DOT = ".$KLIB_METADATA_FILE_EXTENSION"
fun File.unpackZippedKonanLibraryTo(newDir: File) {
// First, run validity checks for the given KLIB file.
zippedKotlinLibraryChecks(this)
if (newDir.exists) {
if (newDir.isDirectory)
newDir.deleteRecursively()
else
newDir.delete()
}
this.withMutableZipFileSystem {
it.file("/").recursiveCopyTo(newDir)
}
check(newDir.exists) { "Could not unpack $this as $newDir." }
}
val List<String>.toUnresolvedLibraries
get() = this.map {
val version = it.substringAfterLast('@', "")
.let { if (it.isEmpty()) null else it }
val name = it.substringBeforeLast('@')
UnresolvedLibrary(name, version)
}
@@ -0,0 +1,10 @@
package org.jetbrains.kotlin.library
data class UnresolvedLibrary(
val path: String,
val libraryVersion: String?) {
fun substitutePath(newPath: String): UnresolvedLibrary {
return UnresolvedLibrary(newPath, libraryVersion)
}
}
@@ -0,0 +1,94 @@
/**
* Copyright 2010-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.konan.file.File
import java.io.RandomAccessFile
import java.nio.channels.FileChannel
data class DeclarationId(val id: Long, val isLocal: Boolean)
class CombinedIrFileReader(file: File) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
init {
val declarationsCount = buffer.int
for (i in 0 until declarationsCount) {
val id = buffer.long
val isLocal = buffer.int != 0
val offset = buffer.int
val size = buffer.int
declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
}
}
fun declarationBytes(id: DeclarationId): ByteArray {
val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
val result = ByteArray(offsetSize.second)
buffer.position(offsetSize.first)
buffer.get(result, 0, offsetSize.second)
return result
}
}
private const val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int).
private const val INDEX_HEADER_SIZE = 4 // sizeof(Int).
class CombinedIrFileWriter(val declarationCount: Int) {
private var currentDeclaration = 0
private var currentPosition = 0
private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
private val randomAccessFile = RandomAccessFile(file.path, "rw")
init {
randomAccessFile.writeInt(declarationCount)
assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
for (i in 0 until declarationCount) {
randomAccessFile.writeLong(-1) // id
randomAccessFile.writeInt(-1) // isLocal
randomAccessFile.writeInt(-1) // offset
randomAccessFile.writeInt(-1) // size
}
currentPosition = randomAccessFile.filePointer.toInt()
assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
}
fun skipDeclaration() {
currentDeclaration++
}
fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
randomAccessFile.writeLong(id.id)
randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
randomAccessFile.writeInt(currentPosition)
randomAccessFile.writeInt(bytes.size)
randomAccessFile.seek(currentPosition.toLong())
randomAccessFile.write(bytes)
assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
currentPosition = randomAccessFile.filePointer.toInt()
currentDeclaration++
}
fun finishWriting(): File {
assert(currentDeclaration == declarationCount)
randomAccessFile.close()
return file
}
}
@@ -0,0 +1,133 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.*
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
open class BaseKotlinLibraryImpl(
private val access: BaseLibraryAccess<KotlinLibraryLayout>,
override val isDefault: Boolean
) : BaseKotlinLibrary {
override val libraryFile get() = access.klib
override val libraryName: String by lazy { access.inPlace { it.libraryName } }
override fun toString() = "$libraryName[default=$isDefault]"
override val manifestProperties: Properties by lazy {
access.inPlace { it.manifestFile.loadProperties() }
}
override val versions: KonanLibraryVersioning by lazy {
manifestProperties.readKonanLibraryVersioning()
}
}
open class MetadataLibraryImpl(
private val access: MetadataLibraryAccess<MetadataKotlinLibraryLayout>
) : MetadataLibrary {
override val moduleHeaderData: ByteArray by lazy {
access.inPlace {
it.moduleHeaderFile.readBytes()
}
}
override fun packageMetadata(fqName: String, partName: String): ByteArray =
access.inPlace {
it.packageFragmentFile(fqName, partName).readBytes()
}
override fun packageMetadataParts(fqName: String): Set<String> =
access.inPlace { inPlaceaccess ->
val fileList =
inPlaceaccess.packageFragmentsDir(fqName)
.listFiles
.mapNotNull {
it.name
.substringBeforeLast(KLIB_METADATA_FILE_EXTENSION_WITH_DOT, missingDelimiterValue = "")
.takeIf { it.isNotEmpty() }
}
fileList.toSortedSet().also {
require(it.size == fileList.size) { "Duplicated names: ${fileList.groupingBy { it }.eachCount().filter { (_, count) -> count > 1 }}" }
}
}
}
open class IrLibraryImpl(
private val access: IrLibraryAccess<IrKotlinLibraryLayout>
) : IrLibrary {
override val irHeader: ByteArray? by lazy {
access.inPlace { library: IrKotlinLibraryLayout ->
library.irHeader.let {
if (it.exists) loadIrHeader() else null
}
}
}
override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal)
private val combinedDeclarations: CombinedIrFileReader by lazy {
CombinedIrFileReader(access.realFiles {
it.irFile
})
}
private fun loadIrHeader(): ByteArray =
access.inPlace {
it.irHeader.readBytes()
}
private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
combinedDeclarations.declarationBytes(DeclarationId(index, isLocal))
override val dataFlowGraph by lazy {
access.inPlace { it: IrKotlinLibraryLayout ->
it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null }
}
}
}
open class BackendLibraryImpl(
base: BaseKotlinLibraryImpl,
metadata: MetadataLibraryImpl,
ir: IrLibraryImpl
) : BackendLibrary,
BaseKotlinLibrary by base,
MetadataLibrary by metadata,
IrLibrary by ir
fun createBackendLibrary(
libraryFile: File,
isDefault: Boolean = false
): BackendLibrary {
val baseAccess = BaseLibraryAccess<KotlinLibraryLayout>(libraryFile)
val metadataAccess = MetadataLibraryAccess<MetadataKotlinLibraryLayout>(libraryFile)
val irAccess = IrLibraryAccess<IrKotlinLibraryLayout>(libraryFile)
val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
val metadata = MetadataLibraryImpl(metadataAccess)
val ir = IrLibraryImpl(irAccess)
return BackendLibraryImpl(base, metadata, ir)
}
@@ -0,0 +1,145 @@
package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.file
import org.jetbrains.kotlin.konan.file.withZipFileSystem
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
import java.nio.file.FileSystem
open class KotlinLibraryLayoutImpl(val klib: File) : KotlinLibraryLayout {
val isZipped = klib.isFile
init {
if (isZipped) zippedKotlinLibraryChecks(klib)
}
override val libDir = if (isZipped) File("/") else klib
override val libraryName
get() =
if (isZipped)
klib.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
else
libDir.path
open val extractingToTemp: KotlinLibraryLayout by lazy {
ExtractingBaseLibraryImpl(this)
}
open fun directlyFromZip(zipFileSystem: FileSystem): KotlinLibraryLayout =
FromZipBaseLibraryImpl(this, zipFileSystem)
}
class MetadataLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), MetadataKotlinLibraryLayout {
override val extractingToTemp: MetadataKotlinLibraryLayout by lazy {
ExtractingMetadataLibraryImpl(this)
}
override fun directlyFromZip(zipFileSystem: FileSystem): MetadataKotlinLibraryLayout =
FromZipMetadataLibraryImpl(this, zipFileSystem)
}
class IrLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), IrKotlinLibraryLayout {
override val extractingToTemp: IrKotlinLibraryLayout by lazy {
ExtractingIrLibraryImpl(this)
}
override fun directlyFromZip(zipFileSystem: FileSystem): IrKotlinLibraryLayout =
FromZipIrLibraryImpl(this, zipFileSystem)
}
open class BaseLibraryAccess<L : KotlinLibraryLayout>(val klib: File) {
open val layout = KotlinLibraryLayoutImpl(klib)
fun <T> realFiles(action: (L) -> T): T =
if (layout.isZipped)
action(layout.extractingToTemp as L)
else
action(layout as L)
fun <T> inPlace(action: (L) -> T): T =
if (layout.isZipped)
layout.klib.withZipFileSystem { zipFileSystem ->
action(layout.directlyFromZip(zipFileSystem) as L)
}
else
action(layout as L)
}
open class MetadataLibraryAccess<L : KotlinLibraryLayout>(klib: File) : BaseLibraryAccess<L>(klib) {
override val layout = MetadataLibraryLayoutImpl(klib)
}
open class IrLibraryAccess<L : KotlinLibraryLayout>(klib: File) : BaseLibraryAccess<L>(klib) {
override val layout = IrLibraryLayoutImpl(klib)
}
open class FromZipBaseLibraryImpl(zipped: KotlinLibraryLayoutImpl, zipFileSystem: FileSystem) :
KotlinLibraryLayout {
override val libraryName = zipped.libraryName
override val libDir = zipFileSystem.file(zipped.libDir)
}
class FromZipMetadataLibraryImpl(zipped: MetadataLibraryLayoutImpl, zipFileSystem: FileSystem) :
FromZipBaseLibraryImpl(zipped, zipFileSystem), MetadataKotlinLibraryLayout
class FromZipIrLibraryImpl(zipped: IrLibraryLayoutImpl, zipFileSystem: FileSystem) :
FromZipBaseLibraryImpl(zipped, zipFileSystem), IrKotlinLibraryLayout
/**
* This class and its children automatically extracts pieces of the library on first access. Use it if you need
* to pass extracted files to an external tool. Otherwise, stick to [FromZipBaseLibraryImpl].
*/
open class KotlinLibraryExtractor(private val zipped: KotlinLibraryLayoutImpl) {
fun extract(file: File): File = zipped.klib.withZipFileSystem { zipFileSystem ->
val temporary = org.jetbrains.kotlin.konan.file.createTempFile(file.name)
zipFileSystem.file(file).copyTo(temporary)
temporary.deleteOnExit()
temporary
}
fun extractDir(directory: File): File = zipped.klib.withZipFileSystem { zipFileSystem ->
val temporary = org.jetbrains.kotlin.konan.file.createTempDir(directory.name)
zipFileSystem.file(directory).recursiveCopyTo(temporary)
temporary.deleteOnExitRecursively()
temporary
}
}
open class ExtractingBaseLibraryImpl(zipped: KotlinLibraryLayoutImpl) :
KotlinLibraryExtractor(zipped),
KotlinLibraryLayout by zipped {
override val manifestFile: File by lazy { extract(zipped.manifestFile) }
override val resourcesDir: File by lazy { extractDir(zipped.resourcesDir) }
}
class ExtractingMetadataLibraryImpl(zipped: MetadataLibraryLayoutImpl) :
KotlinLibraryExtractor(zipped),
MetadataKotlinLibraryLayout by zipped {
override val metadataDir: File by lazy { extractDir(zipped.metadataDir) }
}
class ExtractingIrLibraryImpl(zipped: IrLibraryLayoutImpl) : KotlinLibraryExtractor(zipped),
IrKotlinLibraryLayout by zipped {
override val irFile: File by lazy { extract(zipped.irFile) }
}
internal fun zippedKotlinLibraryChecks(klibFile: File) {
check(klibFile.exists) { "Could not find $klibFile." }
check(klibFile.isFile) { "Expected $klibFile to be a regular file." }
val extension = klibFile.extension
check(extension.isEmpty() || extension == KLIB_FILE_EXTENSION) {
"KLIB path has unexpected extension: $klibFile"
}
}