KLIB reader: better isolation of implementation details (#1917)

KLIB reader:
- better isolation of implementation details
- rename entities closer to their meaning: KonanLibrary -> KonanLibraryLayout, KonanLibraryReader -> KonanLibrary
This commit is contained in:
Dmitriy Dolovov
2018-08-23 10:46:58 +03:00
committed by GitHub
parent a216bf8903
commit 314cd0993b
21 changed files with 391 additions and 329 deletions
-2
View File
@@ -138,8 +138,6 @@ dependencies {
compilerCompile kotlinNativeInterop['llvm'].configuration
compilerCompile kotlinNativeInterop['hash'].configuration
compilerCompile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
compilerCompile "org.jetbrains.kotlin:konan.descriptors:$konanVersion"
compilerCompile "org.jetbrains.kotlin:konan.metadata:$konanVersion"
compilerCompile "org.jetbrains.kotlin:konan.serializer:$konanVersion"
cli_bcCompile kotlinCompilerModule
@@ -32,7 +32,7 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.konan.interop.createForwardDeclarationsModule
import org.jetbrains.kotlin.konan.TempFiles
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.defaultResolver
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.profile
@@ -93,8 +93,10 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
private val repositories = configuration.getList(KonanConfigKeys.REPOSITORIES)
private val resolver = defaultResolver(repositories, target, distribution)
internal val immediateLibraries: List<KonanLibraryReader> by lazy {
val result = resolver.resolveImmediateLibraries(libraryNames, target,
internal val immediateLibraries: List<KonanLibrary> by lazy {
val result = resolver.resolveImmediateLibraries(
libraryNames,
target,
currentAbiVersion,
configuration.getBoolean(KonanConfigKeys.NOSTDLIB),
configuration.getBoolean(KonanConfigKeys.NODEFAULTLIBS),
@@ -103,7 +105,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
result
}
fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List<KonanLibraryReader> {
fun librariesWithDependencies(moduleDescriptor: ModuleDescriptor?): List<KonanLibrary> {
if (moduleDescriptor == null) error("purgeUnneeded() only works correctly after resolve is over, and we have successfully marked package files as needed or not needed.")
return immediateLibraries.purgeUnneeded(this).withResolvedDependencies()
@@ -175,5 +177,5 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
fun CompilerConfiguration.report(priority: CompilerMessageSeverity, message: String)
= this.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(priority, message)
private fun <T: KonanLibraryReader> List<T>.purgeUnneeded(config: KonanConfig): List<T> =
private fun <T: KonanLibrary> List<T>.purgeUnneeded(config: KonanConfig): List<T> =
this.filter{ (!it.isDefaultLibrary && !config.purgeUserLibs) || it.isNeededForLink }
@@ -17,14 +17,14 @@
package org.jetbrains.kotlin.backend.konan.library
import llvm.LLVMModuleRef
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
interface KonanLibraryWriter {
fun addLinkData(linkData: LinkData)
fun addNativeBitcode(library: String)
fun addIncludedBinary(library: String)
fun addKotlinBitcode(llvmModule: LLVMModuleRef)
fun addLinkDependencies(libraries: List<KonanLibraryReader>)
fun addLinkDependencies(libraries: List<KonanLibrary>)
fun addManifestAddend(path: String)
fun addDataFlowGraph(dataFlowGraph: ByteArray)
val mainBitcodeFileName: String
@@ -17,15 +17,16 @@
package org.jetbrains.kotlin.backend.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.SearchPathResolver
import org.jetbrains.kotlin.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.konan.library.createKonanLibraryReader
import org.jetbrains.kotlin.konan.library.unresolvedDependencies
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KONAN_CURRENT_ABI_VERSION = 1
/**
* Returns the list of [KonanLibraryReader]s given the list of user provided [libraryNames] along with
* Returns the list of [KonanLibrary]s given the list of user provided [libraryNames] along with
* other parameters: [target], [abiVersion], [noStdLib], [noDefaultLibs].
*/
fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
@@ -33,13 +34,13 @@ fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List<String>,
abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false,
logger: ((String) -> Unit)?): List<KonanLibraryReader> {
logger: ((String) -> Unit)?): List<KonanLibrary> {
val userProvidedLibraries = libraryNames
.map { resolve(it) }
.map{ LibraryReaderImpl(it, abiVersion, target) }
.map{ createKonanLibraryReader(it, abiVersion, target) }
val defaultLibraries = defaultLinks(noStdLib = noStdLib, noDefaultLibs = noDefaultLibs).map {
LibraryReaderImpl(it, abiVersion, target, isDefaultLibrary = true)
createKonanLibraryReader(it, abiVersion, target, isDefaultLibrary = true)
}
// Make sure the user provided ones appear first, so that
@@ -64,16 +65,16 @@ private fun warnOnLibraryDuplicates(resolvedLibraries: List<File>, logger: ((Str
/**
* For each of the given [immediateLibraries] fills in `resolvedDependencies` field with the
* [KonanLibraryReader]s the library !!directly!! depends on.
* [KonanLibrary]s the library !!directly!! depends on.
*/
fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanLibraryReader>,
fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanLibrary>,
target: KonanTarget,
abiVersion: Int) {
val cache = mutableMapOf<File, KonanLibraryReader>()
val cache = mutableMapOf<File, KonanLibrary>()
cache.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it })
var newDependencies = cache.values.toList()
do {
newDependencies = newDependencies.map { library: KonanLibraryReader ->
newDependencies = newDependencies.map { library: KonanLibrary ->
library.unresolvedDependencies
.map { resolve(it).absoluteFile }
.mapNotNull {
@@ -81,7 +82,7 @@ fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanL
library.resolvedDependencies.add(cache[it]!!)
null
} else {
val reader = LibraryReaderImpl(it, abiVersion, target)
val reader = createKonanLibraryReader(it, abiVersion, target)
cache[it] = reader
library.resolvedDependencies.add(reader)
reader
@@ -92,11 +93,11 @@ fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List<KonanL
}
/**
* For the given list of [KonanLibraryReader]s returns the list of [KonanLibraryReader]s
* For the given list of [KonanLibrary]s returns the list of [KonanLibrary]s
* that includes the same libraries plus all their (transitive) dependencies.
*/
fun List<KonanLibraryReader>.withResolvedDependencies(): List<KonanLibraryReader> {
val result = mutableSetOf<KonanLibraryReader>()
fun List<KonanLibrary>.withResolvedDependencies(): List<KonanLibrary> {
val result = mutableSetOf<KonanLibrary>()
result.addAll(this)
var newDependencies = result.toList()
do {
@@ -112,7 +113,7 @@ fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List<String>,
target: KonanTarget,
abiVersion: Int = KONAN_CURRENT_ABI_VERSION,
noStdLib: Boolean = false,
noDefaultLibs: Boolean = false): List<KonanLibraryReader> {
noDefaultLibs: Boolean = false): List<KonanLibrary> {
val immediateLibraries = resolveImmediateLibraries(
libraryNames = libraryNames,
target = target,
@@ -16,31 +16,34 @@
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import llvm.LLVMModuleRef
import llvm.LLVMWriteBitcodeToFile
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.konan.file.*
import org.jetbrains.kotlin.konan.library.*
import org.jetbrains.kotlin.konan.properties.*
import org.jetbrains.kotlin.konan.target.KonanTarget
abstract class FileBasedLibraryWriter (
val file: File, val currentAbiVersion: Int): KonanLibraryWriter {
}
abstract class FileBasedLibraryWriter (val file: File, val currentAbiVersion: Int): KonanLibraryWriter
class LibraryWriterImpl(override val libDir: File, moduleName: String, currentAbiVersion: Int,
override val target: KonanTarget?, val nopack: Boolean = false):
FileBasedLibraryWriter(libDir, currentAbiVersion), KonanLibrary {
/**
* Requires non-null [target].
*/
class LibraryWriterImpl(
override val libDir: File,
moduleName: String,
currentAbiVersion: Int,
override val target: KonanTarget,
val nopack: Boolean = false
): FileBasedLibraryWriter(libDir, currentAbiVersion), KonanLibraryLayout {
public constructor(path: String, moduleName: String, currentAbiVersion: Int,
target:KonanTarget?, nopack: Boolean):
constructor(path: String, moduleName: String, currentAbiVersion: Int, target: KonanTarget, nopack: Boolean):
this(File(path), moduleName, currentAbiVersion, target, nopack)
override val libraryName = libDir.path
val klibFile
get() = File("${libDir.path}.klib")
get() = File("${libDir.path}.$KLIB_FILE_EXTENSION")
// TODO: Experiment with separate bitcode files.
// Per package or per class.
@@ -60,8 +63,8 @@ class LibraryWriterImpl(override val libDir: File, moduleName: String, currentAb
includedDir.mkdirs()
resourcesDir.mkdirs()
// TODO: <name>:<hash> will go somewhere around here.
manifestProperties.setProperty("unique_name", "$moduleName")
manifestProperties.setProperty("abi_version", "$currentAbiVersion")
manifestProperties.setProperty(KLIB_PROPERTY_UNIQUE_NAME, moduleName)
manifestProperties.setProperty(KLIB_PROPERTY_ABI_VERSION, currentAbiVersion.toString())
}
var llvmModule: LLVMModuleRef? = null
@@ -85,14 +88,14 @@ class LibraryWriterImpl(override val libDir: File, moduleName: String, currentAb
File(library).copyTo(File(includedDir, basename))
}
override fun addLinkDependencies(libraries: List<KonanLibraryReader>) {
override fun addLinkDependencies(libraries: List<KonanLibrary>) {
if (libraries.isEmpty()) {
manifestProperties.remove("depends")
manifestProperties.remove(KLIB_PROPERTY_DEPENDS)
// make sure there are no leftovers from the .def file.
return
} else {
val newValue = libraries .map { it.uniqueName } . joinToString(" ")
manifestProperties.setProperty("depends", newValue)
val newValue = libraries.joinToString(" ") { it.uniqueName }
manifestProperties.setProperty(KLIB_PROPERTY_DEPENDS, newValue)
}
}
@@ -115,18 +118,18 @@ class LibraryWriterImpl(override val libDir: File, moduleName: String, currentAb
}
internal fun buildLibrary(
natives: List<String>,
included: List<String>,
linkDependencies: List<KonanLibraryReader>,
linkData: LinkData,
abiVersion: Int,
target: KonanTarget,
output: String,
moduleName: String,
llvmModule: LLVMModuleRef,
nopack: Boolean,
manifest: String?,
dataFlowGraph: ByteArray?): KonanLibraryWriter {
natives: List<String>,
included: List<String>,
linkDependencies: List<KonanLibrary>,
linkData: LinkData,
abiVersion: Int,
target: KonanTarget,
output: String,
moduleName: String,
llvmModule: LLVMModuleRef,
nopack: Boolean,
manifest: String?,
dataFlowGraph: ByteArray?): KonanLibraryWriter {
val library = LibraryWriterImpl(output, moduleName, abiVersion, target, nopack)
@@ -16,10 +16,10 @@
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
internal class MetadataWriterImpl(library: KonanLibrary): KonanLibrary by library {
internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibraryLayout by libraryLayout {
fun addLinkData(linkData: LinkData) {
moduleHeaderFile.writeBytes(linkData.module)
@@ -20,7 +20,7 @@ import llvm.LLVMTypeRef
import org.jetbrains.kotlin.backend.konan.descriptors.isAbstract
import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
import org.jetbrains.kotlin.backend.konan.isInlined
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PropertyAccessorDescriptor
@@ -30,6 +30,7 @@ import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
import org.jetbrains.kotlin.ir.types.*
import org.jetbrains.kotlin.konan.library.uniqueName
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.constants.StringValue
@@ -322,5 +323,5 @@ internal fun ModuleDescriptor.privateClassSymbolName(index: Int, className: Stri
internal val String.moduleConstructorName
get() = "_Konan_init_${this}"
internal val KonanLibraryReader.moduleConstructorName
internal val KonanLibrary.moduleConstructorName
get() = uniqueName.moduleConstructorName
@@ -25,7 +25,7 @@ import org.jetbrains.kotlin.backend.konan.Context
import org.jetbrains.kotlin.backend.konan.descriptors.findPackage
import org.jetbrains.kotlin.backend.konan.hash.GlobalHash
import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.withResolvedDependencies
import org.jetbrains.kotlin.descriptors.konan.CompiledKonanModuleOrigin
import org.jetbrains.kotlin.descriptors.konan.CurrentKonanModuleOrigin
@@ -323,7 +323,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
return function
}
private val usedLibraries = mutableSetOf<KonanLibraryReader>()
private val usedLibraries = mutableSetOf<KonanLibrary>()
val imports = object : LlvmImports {
@@ -332,7 +332,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
override fun add(origin: CompiledKonanModuleOrigin) {
val reader = when (origin) {
CurrentKonanModuleOrigin -> return
is DeserializedKonanModuleOrigin -> origin.reader as KonanLibraryReader
is DeserializedKonanModuleOrigin -> origin.reader as KonanLibrary
}
if (reader !in allLibraries) {
@@ -343,19 +343,19 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
}
}
val librariesToLink: List<KonanLibraryReader> by lazy {
val librariesToLink: List<KonanLibrary> by lazy {
context.config.immediateLibraries
.filter { (!it.isDefaultLibrary && !context.config.purgeUserLibs) || it in usedLibraries }
.withResolvedDependencies()
.topoSort()
}
private fun List<KonanLibraryReader>.topoSort(): List<KonanLibraryReader> {
var sorted = mutableListOf<KonanLibraryReader>()
val visited = mutableSetOf<KonanLibraryReader>()
val tempMarks = mutableSetOf<KonanLibraryReader>()
private fun List<KonanLibrary>.topoSort(): List<KonanLibrary> {
var sorted = mutableListOf<KonanLibrary>()
val visited = mutableSetOf<KonanLibrary>()
val tempMarks = mutableSetOf<KonanLibrary>()
fun visit(node: KonanLibraryReader, result: MutableList<KonanLibraryReader>) {
fun visit(node: KonanLibrary, result: MutableList<KonanLibrary>) {
if (visited.contains(node)) return
if (tempMarks.contains(node)) error("Cyclic dependency in library graph.")
tempMarks.add(node)
@@ -373,7 +373,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
return sorted
}
val librariesForLibraryManifest: List<KonanLibraryReader> get() {
val librariesForLibraryManifest: List<KonanLibrary> get() {
// Note: library manifest should contain the list of all user libraries and frontend-used default libraries.
// However this would result into linking too many default libraries into the application which uses current
// library. This problem should probably be fixed by adding different kind of dependencies to library
@@ -1,44 +1,64 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.name.FqName
// This scheme describes the Kotlin/Native Library (klib) layout.
interface KonanLibraryLayout {
const val KLIB_PROPERTY_ABI_VERSION = "abi_version"
const val KLIB_PROPERTY_UNIQUE_NAME = "unique_name"
const val KLIB_PROPERTY_LINKED_OPTS = "linkerOpts"
const val KLIB_PROPERTY_DEPENDS = "depends"
const val KLIB_PROPERTY_INTEROP = "interop"
const val KLIB_PROPERTY_PACKAGE = "package"
const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations"
const val KLIB_PROPERTY_INCLUDED_HEADERS = "includedHeaders"
val libDir: File
val target: KonanTarget?
// This is a default implementation. Can't make it an assignment.
get() = null
interface KonanLibrary {
val manifestFile
get() = File(libDir, "manifest")
val resourcesDir
get() = File(libDir, "resources")
val targetsDir
get() = File(libDir, "targets")
val targetDir
get() = File(targetsDir, target!!.visibleName)
val kotlinDir
get() = File(targetDir, "kotlin")
val nativeDir
get() = File(targetDir, "native")
val includedDir
get() = File(targetDir, "included")
val linkdataDir
get() = File(libDir, "linkdata")
val moduleHeaderFile
get() = File(linkdataDir, "module")
val dataFlowGraphFile
get() = File(linkdataDir, "module_data_flow_graph")
fun packageFile(packageName: String)
= File(linkdataDir, if (packageName == "") "root_package.knm" else "package_$packageName.knm")
}
interface KonanLibrary: KonanLibraryLayout {
val libraryName: String
val libraryFile: File
// properties:
val manifestProperties: Properties
val abiVersion: String
val linkerOpts: List<String>
// paths:
val bitcodePaths: List<String>
val includedPaths: List<String>
val targetList: List<String>
val dataFlowGraph: ByteArray?
val moduleHeaderData: ByteArray
fun packageMetadata(fqName: String): ByteArray
// FIXME: ddol: to be refactored into some global resolution context
val isDefaultLibrary: Boolean get() = false
val isNeededForLink: Boolean get() = true
val resolvedDependencies: MutableList<KonanLibrary>
fun markPackageAccessed(fqName: String)
}
val KonanLibrary.uniqueName
get() = manifestProperties.getProperty(KLIB_PROPERTY_UNIQUE_NAME)!!
val KonanLibrary.unresolvedDependencies: List<String>
get() = manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS)
val KonanLibrary.isInterop
get() = manifestProperties.getProperty(KLIB_PROPERTY_INTEROP) == "true"
val KonanLibrary.packageFqName
get() = manifestProperties.getProperty(KLIB_PROPERTY_PACKAGE)?.let { FqName(it) }
val KonanLibrary.exportForwardDeclarations
get() = manifestProperties.getProperty(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS)
.split(' ')
.map { it.trim() }
.filter { it.isNotEmpty() }
.map { FqName(it) }
val KonanLibrary.includedHeaders
get() = manifestProperties.getProperty(KLIB_PROPERTY_INCLUDED_HEADERS).split(' ')
@@ -0,0 +1,45 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.target.KonanTarget
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
// This scheme describes the Kotlin/Native Library (klib) layout.
interface KonanLibraryLayout {
val libraryName: String
val libDir: File
val target: KonanTarget?
// This is a default implementation. Can't make it an assignment.
get() = null
val manifestFile
get() = File(libDir, "manifest")
val resourcesDir
get() = File(libDir, "resources")
val targetsDir
get() = File(libDir, "targets")
val targetDir
get() = File(targetsDir, target!!.visibleName)
val kotlinDir
get() = File(targetDir, "kotlin")
val nativeDir
get() = File(targetDir, "native")
val includedDir
get() = File(targetDir, "included")
val linkdataDir
get() = File(libDir, "linkdata")
val moduleHeaderFile
get() = File(linkdataDir, "module")
val dataFlowGraphFile
get() = File(linkdataDir, "module_data_flow_graph")
fun packageFile(packageName: String)
= File(linkdataDir, if (packageName == "") "root_package.knm" else "package_$packageName.knm")
}
@@ -1,28 +0,0 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.properties.Properties
interface KonanLibraryReader {
val libraryFile: File
val libraryName: String
val manifestProperties: Properties
val uniqueName: String
val linkerOpts: List<String>
val unresolvedDependencies: List<String>
val bitcodePaths: List<String>
val includedPaths: List<String>
val dataFlowGraph: ByteArray?
val moduleHeaderData: ByteArray
fun packageMetadata(fqName: String): ByteArray
val isDefaultLibrary: Boolean get() = false
// FIXME: ddol: to be refactored into some global resolution context
val isNeededForLink: Boolean get() = true
val resolvedDependencies: MutableList<KonanLibraryReader>
fun markPackageAccessed(fqName: String)
}
@@ -0,0 +1,32 @@
package org.jetbrains.kotlin.konan.library
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.impl.DefaultMetadataReaderImpl
import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryChecks
import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryRoot
import org.jetbrains.kotlin.konan.target.KonanTarget
fun File.unpackZippedKonanLibraryTo(newDir: File) {
// first run validity checks for the given KLIB file
zippedKonanLibraryChecks(this)
if (newDir.exists) {
if (newDir.isDirectory)
newDir.deleteRecursively()
else
newDir.delete()
}
zippedKonanLibraryRoot(this).recursiveCopyTo(newDir)
check(newDir.exists) { "Could not unpack $this as $newDir." }
}
fun createKonanLibraryReader(
libraryFile: File,
currentAbiVersion: Int,
target: KonanTarget? = null,
isDefaultLibrary: Boolean = false,
metadataReader: MetadataReader = DefaultMetadataReaderImpl
): KonanLibrary = KonanLibraryImpl(libraryFile, currentAbiVersion, target, isDefaultLibrary, metadataReader)
@@ -1,6 +1,6 @@
package org.jetbrains.kotlin.konan.library
interface MetadataReader {
fun loadSerializedModule(library: KonanLibrary): ByteArray
fun loadSerializedPackageFragment(library: KonanLibrary, fqName: String): ByteArray
fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray
}
@@ -1,12 +1,13 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
import org.jetbrains.kotlin.konan.library.MetadataReader
object DefaultMetadataReaderImpl : MetadataReader {
internal object DefaultMetadataReaderImpl : MetadataReader {
override fun loadSerializedModule(library: KonanLibrary): ByteArray = library.moduleHeaderFile.readBytes()
override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray =
libraryLayout.moduleHeaderFile.readBytes()
override fun loadSerializedPackageFragment(library: KonanLibrary, fqName: String): ByteArray =
library.packageFile(fqName).readBytes()
override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray =
libraryLayout.packageFile(fqName).readBytes()
}
@@ -1,85 +1,79 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_ABI_VERSION
import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_LINKED_OPTS
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.konan.file.*
import org.jetbrains.kotlin.konan.library.MetadataReader
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.serialization.konan.emptyPackages
const val KLIB_FILE_EXTENSION = "klib"
const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
internal class KonanLibraryImpl(
override val libraryFile: File,
private val currentAbiVersion: Int,
internal val target: KonanTarget?,
override val isDefaultLibrary: Boolean,
private val metadataReader: MetadataReader
) : KonanLibrary {
class ZippedKonanLibrary(val klibFile: File, override val target: KonanTarget? = null): KonanLibrary {
init {
check(klibFile.exists) { "Could not find $klibFile." }
check(klibFile.isFile) { "Expected $klibFile to be a regular file." }
// For the zipped libraries inPlace gives files from zip file system
// whereas realFiles extracts them to /tmp.
// For unzipped libraries inPlace and realFiles are the same
// providing files in the library directory.
private val inPlace = createKonanLibraryLayout(libraryFile, target)
private val realFiles = inPlace.realFiles
val extension = klibFile.extension
check(extension.isEmpty() || extension == KLIB_FILE_EXTENSION) { "Unexpected file extension: $extension" }
override val libraryName
get() = inPlace.libraryName
override val manifestProperties: Properties by lazy {
val properties = inPlace.manifestFile.loadProperties()
if (target != null) substitute(properties, defaultTargetSubstitutions(target))
properties
}
override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
override val libDir by lazy { klibFile.asZipRoot }
fun unpackTo(newDir: File) {
if (newDir.exists) {
if (newDir.isDirectory)
newDir.deleteRecursively()
else
newDir.delete()
override val abiVersion: String
get() {
val manifestAbiVersion = manifestProperties.getProperty(KLIB_PROPERTY_ABI_VERSION)
check(currentAbiVersion.toString() == manifestAbiVersion) {
"ABI version mismatch. Compiler expects: $currentAbiVersion, the library is $manifestAbiVersion"
}
return manifestAbiVersion
}
override val linkerOpts: List<String>
get() = manifestProperties.propertyList(KLIB_PROPERTY_LINKED_OPTS, target!!.visibleName)
override val bitcodePaths: List<String>
get() = (realFiles.kotlinDir.listFilesOrEmpty + realFiles.nativeDir.listFilesOrEmpty).map { it.absolutePath }
override val includedPaths: List<String>
get() = realFiles.includedDir.listFilesOrEmpty.map { it.absolutePath }
override val targetList by lazy { inPlace.targetsDir.listFiles.map { it.name } }
override val dataFlowGraph by lazy { inPlace.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } }
override val moduleHeaderData: ByteArray by lazy { metadataReader.loadSerializedModule(inPlace) }
override fun packageMetadata(fqName: String) = metadataReader.loadSerializedPackageFragment(inPlace, fqName)
override var isNeededForLink: Boolean = false
private set
override val resolvedDependencies = mutableListOf<KonanLibrary>()
override fun markPackageAccessed(fqName: String) {
if (!isNeededForLink // fast path
&& !emptyPackages.contains(fqName)) {
isNeededForLink = true
}
libDir.recursiveCopyTo(newDir)
check(newDir.exists) { "Could not unpack $klibFile as $newDir." }
}
private val emptyPackages by lazy { emptyPackages(moduleHeaderData) }
}
// This class automatically extracts pieces of
// the library on first access. Use it if you need
// to pass extracted files to an external tool.
// Otherwise, stick to ZippedKonanLibrary.
private class FileExtractor(zippedLibrary: KonanLibrary): KonanLibrary by zippedLibrary {
override val manifestFile: File by lazy { extract(super.manifestFile) }
override val resourcesDir: File by lazy { extractDir(super.resourcesDir) }
override val includedDir: File by lazy { extractDir(super.includedDir) }
override val kotlinDir: File by lazy { extractDir(super.kotlinDir) }
override val nativeDir: File by lazy { extractDir(super.nativeDir) }
override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
fun extract(file: File): File {
val temporary = createTempFile(file.name)
file.copyTo(temporary)
temporary.deleteOnExit()
return temporary
}
fun extractDir(directory: File): File {
val temporary = createTempDir(directory.name)
directory.recursiveCopyTo(temporary)
temporary.deleteOnExitRecursively()
return temporary
}
}
class UnzippedKonanLibrary(override val libDir: File, override val target: KonanTarget? = null): KonanLibrary {
override val libraryName = libDir.path
val targetList: List<String> by lazy { targetsDir.listFiles.map { it.name } }
}
fun KonanLibrary(klib: File, target: KonanTarget? = null) =
if (klib.isFile) ZippedKonanLibrary(klib, target) else UnzippedKonanLibrary(klib, target)
internal val KonanLibrary.realFiles
get() = when (this) {
is ZippedKonanLibrary -> FileExtractor(this)
// Unpacked library just provides its own files.
is UnzippedKonanLibrary -> this
else -> error("Provide an extractor for your container.")
}
@@ -0,0 +1,79 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.file.asZipRoot
import org.jetbrains.kotlin.konan.file.createTempDir
import org.jetbrains.kotlin.konan.file.createTempFile
import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION
import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION_WITH_DOT
import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget? = null): KonanLibraryLayout {
init { zippedKonanLibraryChecks(klibFile) }
override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
override val libDir by lazy { zippedKonanLibraryRoot(klibFile) }
}
internal fun zippedKonanLibraryChecks(klibFile: File) {
check(klibFile.exists) { "Could not find $klibFile." }
check(klibFile.isFile) { "Expected $klibFile to be a regular file." }
val extension = klibFile.extension
check(extension.isEmpty() || extension == KLIB_FILE_EXTENSION) { "Unexpected file extension: $extension" }
}
internal fun zippedKonanLibraryRoot(klibFile: File) = klibFile.asZipRoot
private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget? = null): KonanLibraryLayout {
override val libraryName = libDir.path
}
// This class automatically extracts pieces of
// the library on first access. Use it if you need
// to pass extracted files to an external tool.
// Otherwise, stick to ZippedKonanLibrary.
private class FileExtractor(zippedLibraryLayout: KonanLibraryLayout): KonanLibraryLayout by zippedLibraryLayout {
override val manifestFile: File by lazy { extract(super.manifestFile) }
override val resourcesDir: File by lazy { extractDir(super.resourcesDir) }
override val includedDir: File by lazy { extractDir(super.includedDir) }
override val kotlinDir: File by lazy { extractDir(super.kotlinDir) }
override val nativeDir: File by lazy { extractDir(super.nativeDir) }
override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
fun extract(file: File): File {
val temporary = createTempFile(file.name)
file.copyTo(temporary)
temporary.deleteOnExit()
return temporary
}
fun extractDir(directory: File): File {
val temporary = createTempDir(directory.name)
directory.recursiveCopyTo(temporary)
temporary.deleteOnExitRecursively()
return temporary
}
}
internal fun createKonanLibraryLayout(klib: File, target: KonanTarget? = null) =
if (klib.isFile) ZippedKonanLibraryLayout(klib, target) else UnzippedKonanLibraryLayout(klib, target)
internal val KonanLibraryLayout.realFiles
get() = when (this) {
is ZippedKonanLibraryLayout -> FileExtractor(this)
// Unpacked library just provides its own files.
is UnzippedKonanLibraryLayout -> this
else -> error("Provide an extractor for your container.")
}
@@ -1,83 +0,0 @@
package org.jetbrains.kotlin.konan.library.impl
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.MetadataReader
import org.jetbrains.kotlin.konan.properties.Properties
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.properties.propertyString
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
import org.jetbrains.kotlin.konan.util.substitute
import org.jetbrains.kotlin.serialization.konan.emptyPackages
class LibraryReaderImpl(
override val libraryFile: File,
val currentAbiVersion: Int,
val target: KonanTarget? = null,
override val isDefaultLibrary: Boolean = false,
private val metadataReader: MetadataReader = DefaultMetadataReaderImpl
) : KonanLibraryReader {
// For the zipped libraries inPlace gives files from zip file system
// whereas realFiles extracts them to /tmp.
// For unzipped libraries inPlace and realFiles are the same
// providing files in the library directory.
private val inPlace = KonanLibrary(libraryFile, target)
private val realFiles = inPlace.realFiles
override val manifestProperties: Properties by lazy {
val properties = inPlace.manifestFile.loadProperties()
if (target != null) substitute(properties, defaultTargetSubstitutions(target))
properties
}
val abiVersion: String
get() {
val manifestAbiVersion = manifestProperties.getProperty("abi_version")
check("$currentAbiVersion" == manifestAbiVersion) {
"ABI version mismatch. Compiler expects: $currentAbiVersion, the library is $manifestAbiVersion"
}
return manifestAbiVersion
}
val targetList = inPlace.targetsDir.listFiles.map { it.name }
override val dataFlowGraph by lazy { inPlace.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } }
override val libraryName
get() = inPlace.libraryName
override val uniqueName
get() = manifestProperties.propertyString("unique_name")!!
override val bitcodePaths: List<String>
get() = (realFiles.kotlinDir.listFilesOrEmpty + realFiles.nativeDir.listFilesOrEmpty).map { it.absolutePath }
override val includedPaths: List<String>
get() = realFiles.includedDir.listFilesOrEmpty.map { it.absolutePath }
override val linkerOpts: List<String>
get() = manifestProperties.propertyList("linkerOpts", target!!.visibleName)
override val unresolvedDependencies: List<String>
get() = manifestProperties.propertyList("depends")
override val resolvedDependencies = mutableListOf<KonanLibraryReader>()
override val moduleHeaderData: ByteArray by lazy { metadataReader.loadSerializedModule(inPlace) }
override var isNeededForLink: Boolean = false
private set
private val emptyPackages by lazy { emptyPackages(moduleHeaderData) }
override fun markPackageAccessed(fqName: String) {
if (!isNeededForLink // fast path
&& !emptyPackages.contains(fqName)) {
isNeededForLink = true
}
}
override fun packageMetadata(fqName: String) = metadataReader.loadSerializedPackageFragment(inPlace, fqName)
}
@@ -7,8 +7,10 @@ import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
import org.jetbrains.kotlin.descriptors.konan.createKonanModuleDescriptor
import org.jetbrains.kotlin.descriptors.konan.interop.InteropFqNames
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.konan.library.exportForwardDeclarations
import org.jetbrains.kotlin.konan.library.isInterop
import org.jetbrains.kotlin.konan.library.packageFqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.serialization.deserialization.*
@@ -19,7 +21,7 @@ import org.jetbrains.kotlin.storage.StorageManager
interface KonanModuleDescriptorFactory {
fun createModuleDescriptor(
libraryReader: KonanLibraryReader,
libraryReader: KonanLibrary,
specifics: LanguageVersionSettings,
storageManager: StorageManager = LockBasedStorageManager()
): ModuleDescriptor
@@ -28,7 +30,7 @@ interface KonanModuleDescriptorFactory {
object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
override fun createModuleDescriptor(
libraryReader: KonanLibraryReader,
libraryReader: KonanLibrary,
specifics: LanguageVersionSettings,
storageManager: StorageManager
): ModuleDescriptorImpl {
@@ -57,7 +59,7 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun createPackageFragmentProvider(
libraryReader: KonanLibraryReader,
libraryReader: KonanLibrary,
fragmentNames: List<String>,
storageManager: StorageManager,
moduleDescriptor: ModuleDescriptor,
@@ -106,21 +108,17 @@ object DefaultKonanModuleDescriptorFactory: KonanModuleDescriptorFactory {
}
private fun getSyntheticPackageFragments(
libraryReader: KonanLibraryReader,
libraryReader: KonanLibrary,
moduleDescriptor: ModuleDescriptor,
konanPackageFragments: List<KonanPackageFragment>
): List<PackageFragmentDescriptor> {
if (libraryReader.manifestProperties.getProperty("interop") != "true") return emptyList()
if (!libraryReader.isInterop) return emptyList()
val packageFqName = libraryReader.manifestProperties.getProperty("package")?.let { FqName(it) }
val packageFqName = libraryReader.packageFqName
?: error("Inconsistent manifest: interop library ${libraryReader.libraryName} should have `package` specified")
val exportForwardDeclarations = libraryReader.manifestProperties.getProperty("exportForwardDeclarations")
.split(' ')
.map { it.trim() }
.filter { it.isNotEmpty() }
.map { FqName(it) }
val exportForwardDeclarations = libraryReader.exportForwardDeclarations
val interopPackageFragments = konanPackageFragments.filter { it.fqName == packageFqName }
@@ -1,7 +1,7 @@
package org.jetbrains.kotlin.serialization.konan
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.konan.library.KonanLibrary
import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
import org.jetbrains.kotlin.name.FqName
@@ -15,7 +15,7 @@ import org.jetbrains.kotlin.storage.StorageManager
class KonanPackageFragment(
val fqNameString: String,
val reader: KonanLibraryReader,
val reader: KonanLibrary,
storageManager: StorageManager,
module: ModuleDescriptor
) : DeserializedPackageFragment(FqName(fqNameString), storageManager, module) {
@@ -24,11 +24,10 @@ import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.konan.isKonanStdlib
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION_WITH_DOT
import org.jetbrains.kotlin.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.konan.library.impl.KLIB_FILE_EXTENSION_WITH_DOT
import org.jetbrains.kotlin.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.konan.library.impl.UnzippedKonanLibrary
import org.jetbrains.kotlin.konan.library.impl.ZippedKonanLibrary
import org.jetbrains.kotlin.konan.library.createKonanLibraryReader
import org.jetbrains.kotlin.konan.library.unpackZippedKonanLibraryTo
import org.jetbrains.kotlin.konan.target.Distribution
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.konan.util.DependencyProcessor
@@ -107,7 +106,7 @@ class Library(val name: String, val requestedRepository: String?, val target: St
val repository = requestedRepository?.File() ?: defaultRepository
fun info() {
val library = libraryInRepoOrCurrentDir(repository, name)
val reader = LibraryReaderImpl(library, currentAbiVersion)
val reader = createKonanLibraryReader(library, currentAbiVersion)
val headerAbiVersion = reader.abiVersion
val moduleName = ModuleDeserializer(reader.moduleHeaderData).moduleName
@@ -122,30 +121,30 @@ class Library(val name: String, val requestedRepository: String?, val target: St
fun install() {
Library(File(name).name.removeSuffix(KLIB_FILE_EXTENSION_WITH_DOT), requestedRepository, target).remove(true)
val library = ZippedKonanLibrary(libraryInCurrentDir(name))
val newLibDir = File(repository, library.libraryName.File().name)
newLibDir.mkdirs()
library.unpackTo(newLibDir)
val klibFile = libraryInCurrentDir(name)
val newLibDir = File(repository, klibFile.name.removeSuffix(KLIB_FILE_EXTENSION_WITH_DOT))
klibFile.unpackZippedKonanLibraryTo(newLibDir)
}
fun remove(blind: Boolean = false) {
if (!repository.exists) error("Repository does not exist: $repository")
val reader = try {
val libDir = try {
val library = libraryInRepo(repository, name)
if (blind) warn("Removing The previously installed $name from $repository.")
UnzippedKonanLibrary(library)
library
} catch (e: Throwable) {
if (!blind) println(e.message)
null
}
reader?.libDir?.deleteRecursively()
libDir?.deleteRecursively()
}
fun contents(output: Appendable = out) {
val reader = LibraryReaderImpl(libraryInRepoOrCurrentDir(repository, name), currentAbiVersion)
val reader = createKonanLibraryReader(libraryInRepoOrCurrentDir(repository, name), currentAbiVersion)
val versionSpec = LanguageVersionSettingsImpl(currentLanguageVersion, currentApiVersion)
val module = DefaultKonanModuleDescriptorFactory.createModuleDescriptor(reader, versionSpec)
val defaultModules = mutableListOf<ModuleDescriptorImpl>()
@@ -158,7 +157,7 @@ class Library(val name: String, val requestedRepository: String?, val target: St
resolver.defaultLinks(false, true)
.mapTo(defaultModules) {
DefaultKonanModuleDescriptorFactory.createModuleDescriptor(
LibraryReaderImpl(it, currentAbiVersion), versionSpec)
createKonanLibraryReader(it, currentAbiVersion), versionSpec)
}
}
@@ -19,6 +19,8 @@ package org.jetbrains.kotlin.cli.utilities
import org.jetbrains.kotlin.backend.konan.library.resolveLibrariesRecursive
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.konan.library.defaultResolver
import org.jetbrains.kotlin.konan.library.includedHeaders
import org.jetbrains.kotlin.konan.library.packageFqName
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.native.interop.gen.jvm.interop
import org.jetbrains.kotlin.utils.addIfNotNull
@@ -71,13 +73,11 @@ fun invokeInterop(flavor: String, args: Array<String>): Array<String> {
libraries, target, noStdLib = true, noDefaultLibs = noDefaultLibs
)
val importArgs = allLibraries.flatMap {
val manifestProperties = it.manifestProperties
val importArgs = allLibraries.flatMap {libraryReader ->
// TODO: handle missing properties?
manifestProperties["package"]?.let {
val pkg = it as String
val headerIds = (manifestProperties["includedHeaders"] as String).split(' ')
val arg = "$pkg:${headerIds.joinToString(";")}"
libraryReader.packageFqName?.asString()?.let { packageFqName ->
val headerIds = libraryReader.includedHeaders
val arg = "$packageFqName:${headerIds.joinToString(";")}"
listOf("-import", arg)
} ?: emptyList()
}