Konan Library reshuffle.

Distributed classes into several files.
Eliminated Base64 encoding of protobufs, it was a kt.bc library leftover.
This commit is contained in:
Alexander Gorshenev
2017-06-23 13:50:38 +03:00
committed by alexander-gorshenev
parent 07ba122d96
commit e257445a3b
21 changed files with 414 additions and 554 deletions
@@ -18,7 +18,7 @@ package org.jetbrains.kotlin.backend.konan
import com.intellij.openapi.project.Project
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.backend.konan.library.SplitLibraryReader
import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.backend.konan.util.profile
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
@@ -33,6 +33,8 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind.*
class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
val currentAbiVersion = configuration.get(KonanConfigKeys.ABI_VERSION)!!
internal val targetManager = TargetManager(
configuration.get(KonanConfigKeys.TARGET))
@@ -71,10 +73,9 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
}
internal val libraries: List<KonanLibraryReader> by lazy {
val currentAbiVersion = configuration.get(KonanConfigKeys.ABI_VERSION)!!
val target = targetManager.targetName
// Here we have chosen a particular KonanLibraryReader implementation.
librariesFound.map{it -> SplitLibraryReader(it, currentAbiVersion, target)}
librariesFound.map{it -> LibraryReaderImpl(it, currentAbiVersion, target)}
}
private val loadedDescriptors = loadLibMetadata()
@@ -89,7 +90,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
for (klib in libraries) {
profile("Loading ${klib.libraryName}") {
val moduleDescriptor = klib.moduleDescriptor(specifics)
// MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor.
val moduleDescriptor = klib.moduleDescriptor(specifics) as ModuleDescriptorImpl
allMetadata.add(moduleDescriptor)
}
}
@@ -33,6 +33,8 @@ class KonanConfigKeys {
= CompilerConfigurationKey.create("enable runtime assertions in generated code")
val ENABLED_PHASES: CompilerConfigurationKey<List<String>>
= CompilerConfigurationKey.create("enable backend phases")
val ENTRY: CompilerConfigurationKey<String?>
= CompilerConfigurationKey.create("fully qualified main() name")
val LIBRARY_FILES: CompilerConfigurationKey<List<String>>
= CompilerConfigurationKey.create("library file paths")
val LINKER_ARGS: CompilerConfigurationKey<List<String>>
@@ -41,8 +43,6 @@ class KonanConfigKeys {
= CompilerConfigurationKey.create("list backend phases")
val LIST_TARGETS: CompilerConfigurationKey<Boolean>
= CompilerConfigurationKey.create("list available targets")
val ENTRY: CompilerConfigurationKey<String?>
= CompilerConfigurationKey.create("fully qualified main() name")
val META_INFO: CompilerConfigurationKey<List<String>>
= CompilerConfigurationKey.create("generate metadata")
val MODULE_KIND: CompilerConfigurationKey<ModuleKind>
@@ -16,8 +16,7 @@
package org.jetbrains.kotlin.backend.konan
import java.io.File
import java.util.Properties
import org.jetbrains.kotlin.backend.konan.util.*
public class KonanProperties(val propertyFile: String) {
@@ -16,52 +16,10 @@
package org.jetbrains.kotlin.backend.konan.library
import llvm.LLVMModuleRef
import llvm.LLVMWriteBitcodeToFile
import org.jetbrains.kotlin.backend.konan.KonanConfigKeys
import org.jetbrains.kotlin.backend.konan.serialization.Base64
import org.jetbrains.kotlin.backend.konan.serialization.deserializeModule
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.copyTo
import org.jetbrains.kotlin.backend.konan.util.unzipAs
import org.jetbrains.kotlin.backend.konan.util.zipDirAs
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
interface KonanLibraryReader {
val libraryName: String
val moduleName: String
val bitcodePaths: List<String>
fun moduleDescriptor(specifics: LanguageVersionSettings): ModuleDescriptorImpl
}
abstract open class FileBasedLibraryReader(
val file: File, val currentAbiVersion: Int,
val reader: MetadataReader): KonanLibraryReader {
override val libraryName: String
get() = file.path
protected val namedModuleData: NamedModuleData by lazy {
reader.loadSerializedModule(currentAbiVersion)
}
override val moduleName: String
get() = namedModuleData.name
val tableOfContents : Base64
get() = namedModuleData.base64
fun packageMetadata(fqName: String): Base64 =
reader.loadSerializedPackageFragment(fqName)
override fun moduleDescriptor(specifics: LanguageVersionSettings)
= deserializeModule(specifics, {packageMetadata(it)},
tableOfContents, moduleName)
}
// This scheme describes the Konan Library (klib) layout.
interface SplitLibraryScheme {
interface KonanLibrary {
val libDir: File
val target: String?
// This is a default implementation. Can't make it an assignment.
@@ -88,107 +46,3 @@ interface SplitLibraryScheme {
= File(linkdataDir, if (packageName == "") "root_package" else "package_$packageName")
}
class SplitLibraryReader(override val libDir: File, currentAbiVersion: Int,
override val target: String?) :
FileBasedLibraryReader(libDir, currentAbiVersion, SplitMetadataReader(libDir)),
SplitLibraryScheme {
public constructor(path: String, currentAbiVersion: Int, target: String?) :
this(File(path), currentAbiVersion, target)
init {
unpackIfNeeded()
}
// TODO: Search path processing is also goes somewhere around here.
fun unpackIfNeeded() {
// TODO: Clarify the policy here.
if (libDir.exists) {
if (libDir.isDirectory) return
}
if (!klibFile.exists) {
error("Could not find neither $libDir nor $klibFile.")
}
if (klibFile.isFile) {
klibFile.unzipAs(libDir)
if (!libDir.exists) error("Could not unpack $klibFile as $libDir.")
} else {
error("Expected $klibFile to be a regular file.")
}
}
override val bitcodePaths: List<String>
get() = (kotlinDir.listFiles + nativeDir.listFiles).map{it.absolutePath}
}
/* ------------ writer part ----------------*/
interface KonanLibraryWriter {
fun addLinkData(linkData: LinkData)
fun addNativeBitcode(library: String)
fun addKotlinBitcode(llvmModule: LLVMModuleRef)
fun commit()
val mainBitcodeFileName: String
}
class LinkData(
val abiVersion: Int,
val module: String,
val moduleName: String,
val fragments: List<String>,
val fragmentNames: List<String> )
abstract class FileBasedLibraryWriter (
val file: File): KonanLibraryWriter {
}
class SplitLibraryWriter(override val libDir: File, override val target: String?,
val nopack: Boolean = false): FileBasedLibraryWriter(libDir), SplitLibraryScheme {
public constructor(path: String, target: String, nopack: Boolean):
this(File(path), target, nopack)
// TODO: Experiment with separate bitcode files.
// Per package or per class.
val mainBitcodeFile = File(kotlinDir, "program.kt.bc")
override val mainBitcodeFileName = mainBitcodeFile.path
init {
// TODO: figure out the proper policy here.
libDir.deleteRecursively()
klibFile.delete()
libDir.mkdirs()
linkdataDir.mkdirs()
targetDir.mkdirs()
kotlinDir.mkdirs()
nativeDir.mkdirs()
resourcesDir.mkdirs()
}
var llvmModule: LLVMModuleRef? = null
override fun addKotlinBitcode(llvmModule: LLVMModuleRef) {
this.llvmModule = llvmModule
LLVMWriteBitcodeToFile(llvmModule, mainBitcodeFileName)
}
override fun addLinkData(linkData: LinkData) {
SplitMetadataGenerator(libDir).addLinkData(linkData)
}
override fun addNativeBitcode(library: String) {
val basename = File(library).name
File(library).copyTo(File(nativeDir, basename))
}
override fun commit() {
if (!nopack) {
libDir.zipDirAs(klibFile)
libDir.deleteRecursively()
}
}
}
@@ -0,0 +1,31 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
interface KonanLibraryReader {
val libraryName: String
val bitcodePaths: List<String>
fun moduleDescriptor(specifics: LanguageVersionSettings): ModuleDescriptor
}
interface MetadataReader {
fun loadSerializedModule(): ByteArray
fun loadSerializedPackageFragment(fqName: String): ByteArray
}
@@ -0,0 +1,37 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import llvm.LLVMModuleRef
interface KonanLibraryWriter {
fun addLinkData(linkData: LinkData)
fun addNativeBitcode(library: String)
fun addKotlinBitcode(llvmModule: LLVMModuleRef)
val mainBitcodeFileName: String
fun commit()
}
class LinkData(
val module: ByteArray,
val fragments: List<ByteArray>,
val fragmentNames: List<String>
)
interface MetadataWriter {
fun addLinkData(linkData: LinkData)
}
@@ -1,73 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import llvm.LLVMLinkModules2
import llvm.LLVMModuleRef
import llvm.LLVMWriteBitcodeToFile
import org.jetbrains.kotlin.backend.konan.llvm.*
import org.jetbrains.kotlin.backend.konan.serialization.Base64
import org.jetbrains.kotlin.backend.konan.serialization.deserializeModule
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.copyTo
import org.jetbrains.kotlin.backend.konan.util.unzipAs
import org.jetbrains.kotlin.backend.konan.util.zipDirAs
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
class KtBcLibraryReader(file: File, currentAbiVersion: Int)
: FileBasedLibraryReader(file, currentAbiVersion, KtBcMetadataReader(file)) {
public constructor(path: String, currentAbiVersion: Int) : this(File(path), currentAbiVersion)
override val bitcodePaths: List<String>
get() = listOf(libraryName)
}
class KtBcLibraryWriter(file: File, val llvmModule: LLVMModuleRef)
: FileBasedLibraryWriter(file) {
override val mainBitcodeFileName = file.path
public constructor(path: String, llvmModule: LLVMModuleRef)
: this(File(path), llvmModule)
override fun addKotlinBitcode(llvmModule: LLVMModuleRef) {
// This is a noop for .kt.bc based libraries,
// because the bitcode itself is the container.
}
override fun addLinkData(linkData: LinkData) {
KtBcMetadataGenerator(llvmModule).addLinkData(linkData)
}
override fun addNativeBitcode(library: String) {
val libraryModule = parseBitcodeFile(library)
val failed = LLVMLinkModules2(llvmModule, libraryModule)
if (failed != 0) {
throw Error("failed to link $library") // TODO: retrieve error message from LLVM.
}
}
override fun commit() {
LLVMWriteBitcodeToFile(llvmModule, file.path)
}
}
@@ -1,184 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import kotlinx.cinterop.*
import llvm.*
import org.jetbrains.kotlin.backend.konan.util.File
import java.io.Closeable
import java.util.Properties
// We still keep this format around for now. But, most probably, it will
// go away completely soon.
class KtBcMetadataReader(file: File) : Closeable, MetadataReader {
override fun loadSerializedModule(currentAbiVersion: Int): NamedModuleData {
val (nodeCount, kmetadataNodeArg) = namedMetadataNode("kmetadata")
if (nodeCount != 1) {
throw Error("Unknown metadata format. The 'kmetadata' node has ${nodeCount} arguments. Don't know what to do.")
}
val operands = metadataNodeOperands(kmetadataNodeArg)
val abiNode = operands[0]
val nameNode = operands[1]
val dataNode = operands[2]
val abiVersion = string(abiNode).toInt()
if (abiVersion != currentAbiVersion) {
throw Error("Expected ABI version ${currentAbiVersion}, but the binary is ${abiVersion}")
}
val moduleName = string(nameNode)
val tableOfContents = string(dataNode)
return NamedModuleData(moduleName, tableOfContents)
}
override fun loadSerializedPackageFragment(fqName: String): String {
val (nodeCount, kpackageNodeArg) = namedMetadataNode("kpackage:$fqName")
if (nodeCount != 1) {
throw Error("The 'kpackage:$fqName' node has ${nodeCount} arguments.")
}
val operands = metadataNodeOperands(kpackageNodeArg)
val dataNode = operands[0]
val base64 = string(dataNode)
return base64
}
lateinit var llvmModule: LLVMModuleRef
lateinit var llvmContext: LLVMContextRef
init {
memScoped {
val bufRef = alloc<LLVMMemoryBufferRefVar>()
val errorRef = allocPointerTo<ByteVar>()
val res = LLVMCreateMemoryBufferWithContentsOfFile(file.toString(), bufRef.ptr, errorRef.ptr)
if (res != 0) {
throw Error(errorRef.value?.toKString())
}
llvmContext = LLVMContextCreate()!!
val moduleRef = alloc<LLVMModuleRefVar>()
val parseResult = LLVMParseBitcodeInContext2(llvmContext, bufRef.value, moduleRef.ptr)
if (parseResult != 0) {
throw Error(parseResult.toString())
}
llvmModule = moduleRef.value!!
}
}
fun string(node: LLVMValueRef): String {
memScoped {
val len = alloc<IntVar>()
val str1 = LLVMGetMDString(node, len.ptr)!!
val str = str1.toKString()
return str
}
}
fun namedMetadataNodes(name: String): Array<LLVMValueRef> {
val result = mutableListOf<LLVMValueRef>()
memScoped {
val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, name)
val nodeArray = allocArray<LLVMValueRefVar>(nodeCount)
LLVMGetNamedMetadataOperands(llvmModule, name, nodeArray)
//return Pair(nodeCount, nodeArray[0].value!!)
for (index in 0..nodeCount-1) {
result.add(nodeArray[index]!!)
}
}
return result.toTypedArray<LLVMValueRef>()
}
fun namedMetadataNode(name: String): Pair<Int, LLVMValueRef> {
memScoped {
val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, name)
val nodeArray = allocArray<LLVMValueRefVar>(nodeCount)
LLVMGetNamedMetadataOperands(llvmModule, name, nodeArray)
return Pair(nodeCount, nodeArray[0]!!)
}
}
fun metadataNodeOperands(metadataNode: LLVMValueRef): Array<LLVMValueRef> {
memScoped {
val operandCount = LLVMGetMDNodeNumOperands(metadataNode)
val operandArray = allocArray<LLVMValueRefVar>(operandCount)
LLVMGetMDNodeOperands(metadataNode, operandArray)
return Array(operandCount, {index -> operandArray[index]!!})
}
}
override fun close() {
LLVMDisposeModule(llvmModule)
LLVMContextDispose(llvmContext)
}
}
internal class KtBcMetadataGenerator(val llvmModule: LLVMModuleRef) {
private fun metadataNode(args: List<LLVMValueRef?>): LLVMValueRef {
return LLVMMDNode(args.toCValues(), args.size)!!
}
private fun metadataFun(fn: LLVMValueRef, info: String): LLVMValueRef {
val args = listOf(fn, metadataString(info));
val md = metadataNode(args)
return md
}
private fun emitModuleMetadata(name: String, md: LLVMValueRef?) {
LLVMAddNamedMetadataOperand(llvmModule, name, md)
}
private fun metadataString(str: String): LLVMValueRef {
return LLVMMDString(str, str.length)!!
}
fun addLinkData(linkData: LinkData) {
if (linkData == null) return
val abiNode = metadataString("${linkData.abiVersion}")
val moduleNameNode = metadataString(linkData.moduleName)
val module = linkData.module
val fragments = linkData.fragments
val fragmentNames = linkData.fragmentNames
val dataNode = metadataString(module)
val kmetadataArg = metadataNode(listOf(abiNode, moduleNameNode, dataNode))
emitModuleMetadata("kmetadata", kmetadataArg)
fragments.forEachIndexed { index, it ->
val name = fragmentNames[index]
val dataNode = metadataString(it)
val kpackageArg = metadataNode(listOf(dataNode))
emitModuleMetadata("kpackage:$name", kpackageArg)
}
}
}
@@ -1,71 +0,0 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library
import org.jetbrains.kotlin.backend.konan.util.File
import java.util.Properties
class NamedModuleData(val name:String, val base64: String)
interface MetadataReader {
fun loadSerializedModule(currentAbiVersion: Int): NamedModuleData
fun loadSerializedPackageFragment(fqName: String): String
}
class SplitMetadataReader(override val libDir: File) : MetadataReader, SplitLibraryScheme {
override fun loadSerializedModule(currentAbiVersion: Int): NamedModuleData {
val header = Properties()
manifestFile.bufferedReader().use { reader ->
header.load(reader)
}
val headerAbiVersion = header.getProperty("abi_version")!!
val moduleName = header.getProperty("module_name")!!
val moduleData = moduleHeaderFile.readText()
if ("$currentAbiVersion" != headerAbiVersion)
error("ABI version mismatch. Compiler expects: $currentAbiVersion, the library is $headerAbiVersion")
return NamedModuleData(moduleName, moduleData)
}
override fun loadSerializedPackageFragment(fqName: String)
= packageFile(fqName).readText()
}
internal class SplitMetadataGenerator(override val libDir: File): SplitLibraryScheme {
fun addLinkData(linkData: LinkData) {
val linkdataDir = File(libDir, "linkdata")
val header = Properties()
header.putAll(hashMapOf(
"abi_version" to "${linkData.abiVersion}",
"module_name" to "${linkData.moduleName}"
))
moduleHeaderFile.writeText(linkData.module)
manifestFile.outputStream().use {
header.store(it, null)
}
linkData.fragments.forEachIndexed { index, it ->
val name = linkData.fragmentNames[index]
packageFile(name).writeText(it)
}
}
}
@@ -17,8 +17,8 @@
package org.jetbrains.kotlin.backend.konan.library
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
interface SearchPathResolver {
val searchRoots: List<File>
@@ -0,0 +1,95 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.backend.konan.library.MetadataReader
import org.jetbrains.kotlin.backend.konan.serialization.deserializeModule
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.loadProperties
import org.jetbrains.kotlin.backend.konan.util.Properties
import org.jetbrains.kotlin.backend.konan.util.unzipAs
import org.jetbrains.kotlin.config.LanguageVersionSettings
abstract class FileBasedLibraryReader(
val file: File, val currentAbiVersion: Int,
val reader: MetadataReader): KonanLibraryReader {
override val libraryName: String
get() = file.path
val moduleHeaderData: ByteArray by lazy {
reader.loadSerializedModule()
}
fun packageMetadata(fqName: String): ByteArray =
reader.loadSerializedPackageFragment(fqName)
override fun moduleDescriptor(specifics: LanguageVersionSettings)
= deserializeModule(specifics, {packageMetadata(it)},
moduleHeaderData)
}
class LibraryReaderImpl(override val libDir: File, currentAbiVersion: Int,
override val target: String?) :
FileBasedLibraryReader(libDir, currentAbiVersion, MetadataReaderImpl(libDir)),
KonanLibrary {
public constructor(path: String, currentAbiVersion: Int, target: String?) :
this(File(path), currentAbiVersion, target)
init {
unpackIfNeeded()
}
// TODO: Search path processing is also goes somewhere around here.
fun unpackIfNeeded() {
// TODO: Clarify the policy here.
if (libDir.exists) {
if (libDir.isDirectory) return
}
if (!klibFile.exists) {
error("Could not find neither $libDir nor $klibFile.")
}
if (klibFile.isFile) {
klibFile.unzipAs(libDir)
if (!libDir.exists) error("Could not unpack $klibFile as $libDir.")
} else {
error("Expected $klibFile to be a regular file.")
}
}
val manifestProperties: Properties by lazy {
manifestFile.loadProperties()
}
val abiVersion: String
get() {
val manifestAbiVersion = manifestProperties.getProperty("abi_version")
if ("$currentAbiVersion" != manifestAbiVersion)
error("ABI version mismatch. Compiler expects: $currentAbiVersion, the library is $manifestAbiVersion")
return manifestAbiVersion
}
override val bitcodePaths: List<String>
get() = (kotlinDir.listFiles + nativeDir.listFiles).map{it.absolutePath}
}
@@ -0,0 +1,96 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library.impl
import llvm.LLVMModuleRef
import llvm.LLVMWriteBitcodeToFile
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.backend.konan.util.*
abstract class FileBasedLibraryWriter (
val file: File, val currentAbiVersion: Int): KonanLibraryWriter {
}
class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int,
override val target: String?, val nopack: Boolean = false):
FileBasedLibraryWriter(libDir, currentAbiVersion), KonanLibrary {
public constructor(path: String, currentAbiVersion: Int,
target: String, nopack: Boolean):
this(File(path), currentAbiVersion, target, nopack)
// TODO: Experiment with separate bitcode files.
// Per package or per class.
val mainBitcodeFile = File(kotlinDir, "program.kt.bc")
override val mainBitcodeFileName = mainBitcodeFile.path
val manifestProperties = Properties()
init {
// TODO: figure out the proper policy here.
libDir.deleteRecursively()
klibFile.delete()
libDir.mkdirs()
linkdataDir.mkdirs()
targetDir.mkdirs()
kotlinDir.mkdirs()
nativeDir.mkdirs()
resourcesDir.mkdirs()
manifestProperties.setProperty("abi_version", "$currentAbiVersion")
}
var llvmModule: LLVMModuleRef? = null
override fun addKotlinBitcode(llvmModule: LLVMModuleRef) {
this.llvmModule = llvmModule
LLVMWriteBitcodeToFile(llvmModule, mainBitcodeFileName)
}
override fun addLinkData(linkData: LinkData) {
MetadataWriterImpl(libDir).addLinkData(linkData)
}
override fun addNativeBitcode(library: String) {
val basename = File(library).name
File(library).copyTo(File(nativeDir, basename))
}
override fun commit() {
manifestProperties.saveToFile(manifestFile)
if (!nopack) {
libDir.zipDirAs(klibFile)
libDir.deleteRecursively()
}
}
}
internal fun buildLibrary(natives: List<String>, linkData: LinkData, abiVersion: Int, target: String, output: String, llvmModule: LLVMModuleRef, nopack: Boolean): KonanLibraryWriter {
val library = LibraryWriterImpl(output, abiVersion, target, nopack)
library.addKotlinBitcode(llvmModule)
library.addLinkData(linkData)
natives.forEach {
library.addNativeBitcode(it)
}
library.commit()
return library
}
@@ -0,0 +1,31 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.MetadataReader
import org.jetbrains.kotlin.backend.konan.util.File
class MetadataReaderImpl(override val libDir: File) : MetadataReader, KonanLibrary {
override fun loadSerializedModule(): ByteArray {
return moduleHeaderFile.readBytes()
}
override fun loadSerializedPackageFragment(fqName: String)
= packageFile(fqName).readBytes()
}
@@ -0,0 +1,35 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.backend.konan.util.File
internal class MetadataWriterImpl(override val libDir: File): KonanLibrary {
fun addLinkData(linkData: LinkData) {
val linkdataDir = File(libDir, "linkdata")
moduleHeaderFile.writeBytes(linkData.module)
linkData.fragments.forEachIndexed { index, it ->
val name = linkData.fragmentNames[index]
packageFile(name).writeBytes(it)
}
}
}
@@ -22,11 +22,11 @@ import org.jetbrains.kotlin.backend.common.descriptors.allParameters
import org.jetbrains.kotlin.backend.common.descriptors.isSuspend
import org.jetbrains.kotlin.backend.common.ir.ir2string
import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.backend.konan.descriptors.*
import org.jetbrains.kotlin.backend.konan.ir.*
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
import org.jetbrains.kotlin.backend.konan.library.SplitLibraryWriter
import org.jetbrains.kotlin.backend.konan.library.impl.buildLibrary
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.incremental.components.NoLookupLocation
@@ -109,12 +109,14 @@ internal fun produceOutput(context: Context) {
}
CompilerOutputKind.LIBRARY -> {
val libraryName = context.config.outputName
val nopack = config.getBoolean(KonanConfigKeys.NOPACK)
val abiVersion = context.config.currentAbiVersion
val targetName = context.config.targetManager.targetName
val nopack = config.getBoolean(KonanConfigKeys.NOPACK)
val library = buildLibrary(
context.config.nativeLibraries,
context.serializedLinkData!!,
abiVersion,
targetName,
libraryName,
llvmModule,
@@ -131,22 +133,6 @@ internal fun produceOutput(context: Context) {
}
}
internal fun buildLibrary(natives: List<String>, linkData: LinkData, target: String, output: String, llvmModule: LLVMModuleRef, nopack: Boolean): KonanLibraryWriter {
// TODO: May be we need a factory?
//val library = KtBcLibraryWriter(output, llvmModule)
val library = SplitLibraryWriter(output, target, nopack)
library.addKotlinBitcode(llvmModule)
library.addLinkData(linkData)
natives.forEach {
library.addNativeBitcode(it)
}
library.commit()
return library
}
internal fun verifyModule(llvmModule: LLVMModuleRef, current: String = "") {
memScoped {
val errorRef = allocPointerTo<ByteVar>()
@@ -89,6 +89,7 @@ message Classes {
}
message Library {
repeated string package_fragment_name = 1;
required string module_name = 1;
repeated string package_fragment_name = 2;
}
@@ -65,24 +65,6 @@ import java.io.ByteArrayOutputStream
* with MemberDeserializer class.
*/
typealias Base64 = String
fun byteArrayToBase64(byteArray: ByteArray): Base64 {
val gzipped = ByteArrayOutputStream()
val gzipStream = GZIPOutputStream(gzipped)
gzipStream.write(byteArray)
gzipStream.close()
val base64 = base64Encode(gzipped.toByteArray())
return base64
}
fun base64ToStream(base64: Base64): InputStream {
val gzipped = base64Decode(base64)
return GZIPInputStream(ByteArrayInputStream(gzipped))
}
/* ------------ Deserializer part ------------------------------------------*/
object NullFlexibleTypeDeserializer : FlexibleTypeDeserializer {
@@ -124,16 +106,19 @@ fun createKonanPackageFragmentProvider(
return provider
}
public fun parsePackageFragment(base64: Base64): PackageFragment =
PackageFragment.parseFrom(base64ToStream(base64),
public fun parsePackageFragment(packageData: ByteArray): PackageFragment =
PackageFragment.parseFrom(packageData,
KonanSerializerProtocol.extensionRegistry)
public fun parseModuleHeader(base64: Base64): Library =
Library.parseFrom(base64ToStream(base64),
public fun parseModuleHeader(libraryData: ByteArray): Library =
Library.parseFrom(libraryData,
KonanSerializerProtocol.extensionRegistry)
internal fun deserializeModule(languageVersionSettings: LanguageVersionSettings,
packageLoader:(String)->Base64, library: Base64, moduleName: String): ModuleDescriptorImpl {
packageLoader:(String)->ByteArray, library: ByteArray): ModuleDescriptorImpl {
val libraryProto = parseModuleHeader(library)
val moduleName = libraryProto.moduleName
val storageManager = LockBasedStorageManager()
val builtIns = KonanBuiltIns(storageManager)
@@ -142,8 +127,6 @@ internal fun deserializeModule(languageVersionSettings: LanguageVersionSettings,
builtIns.builtInsModule = moduleDescriptor
val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings)
val libraryProto = parseModuleHeader(library)
val provider = createKonanPackageFragmentProvider(
libraryProto.packageFragmentNameList,
{it -> parsePackageFragment(packageLoader(it))},
@@ -264,7 +247,8 @@ internal class KonanSerializationUtil(val context: Context) {
}
internal fun serializeModule(moduleDescriptor: ModuleDescriptor): LinkData {
val libraryProto = KonanLinkData.Library.newBuilder()
val fragments = mutableListOf<String>()
libraryProto.moduleName = moduleDescriptor.name.asString()
val fragments = mutableListOf<ByteArray>()
val fragmentNames = mutableListOf<String>()
getPackagesFqNames(moduleDescriptor).forEach iteration@ {
@@ -272,15 +256,11 @@ internal class KonanSerializationUtil(val context: Context) {
if (packageProto == null) return@iteration
libraryProto.addPackageFragmentName(it.asString())
fragments.add(
byteArrayToBase64(packageProto.toByteArray()))
fragments.add(packageProto.toByteArray())
fragmentNames.add(it.asString())
}
val libraryAsByteArray = libraryProto.build().toByteArray()
val library = byteArrayToBase64(libraryAsByteArray)
val abiVersion = context.config.configuration.get(KonanConfigKeys.ABI_VERSION)!!
val moduleName = moduleDescriptor.name.asString()
return LinkData(abiVersion, library, moduleName, fragments, fragmentNames)
return LinkData(libraryAsByteArray, fragments, fragmentNames)
}
}
@@ -56,7 +56,9 @@ class File(val path: String) {
fun delete() = javaFile.delete()
fun deleteRecursively() = javaFile.deleteRecursively()
fun readText() = javaFile.readText()
fun readBytes() = javaFile.readBytes()
fun writeText(text: String) = javaFile.writeText(text)
fun writeBytes(bytes: ByteArray) = javaFile.writeBytes(bytes)
override fun toString() = path
@@ -0,0 +1,35 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.util
typealias Properties = java.util.Properties
fun File.loadProperties(): Properties {
val properties = java.util.Properties()
this.bufferedReader().use { reader ->
properties.load(reader)
}
return properties
}
fun File.saveProperties(properties: Properties) {
this.outputStream().use {
properties.store(it, null)
}
}
fun Properties.saveToFile(file: File) = file.saveProperties(this)
@@ -17,7 +17,6 @@
package org.jetbrains.kotlin.cli.klib
import org.jetbrains.kotlin.serialization.KonanLinkData
import org.jetbrains.kotlin.backend.konan.serialization.Base64
import org.jetbrains.kotlin.backend.konan.serialization.parseModuleHeader
import org.jetbrains.kotlin.backend.konan.serialization.parsePackageFragment
import org.jetbrains.kotlin.backend.konan.serialization.KonanSerializerProtocol
@@ -26,17 +25,24 @@ import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.utils.Printer
import java.lang.System.out
class PrettyPrinter(val library: Base64, val packageLoader: (String) -> Base64) {
private val moduleHeader: KonanLinkData.Library
open class ModuleDeserializer(val library: ByteArray) {
protected val moduleHeader: KonanLinkData.Library
get() = parseModuleHeader(library)
fun packageFragment(fqname: String): KonanLinkData.PackageFragment
= parsePackageFragment(packageLoader(fqname))
val moduleName: String
get() = moduleHeader.moduleName
val packageFragmentNameList: List<String>
get() = moduleHeader.packageFragmentNameList
}
class PrettyPrinter(library: ByteArray, val packageLoader: (String) -> ByteArray)
: ModuleDeserializer(library) {
private fun packageFragment(fqname: String): KonanLinkData.PackageFragment
= parsePackageFragment(packageLoader(fqname))
fun printPackageFragment(fqname: String) {
if (fqname.isNotEmpty()) println("\npackage $fqname")
val fragment = packageFragment(fqname)
@@ -576,4 +582,4 @@ class PackageFragmentPrinter(val packageFragment: KonanLinkData.PackageFragment,
}
return buff.toString()
}
}
}
@@ -19,8 +19,7 @@ package org.jetbrains.kotlin.cli.klib
import kotlin.system.exitProcess
import java.util.Properties
// TODO: Extract these as a shared jar?
import org.jetbrains.kotlin.backend.konan.library.SplitLibraryScheme
import org.jetbrains.kotlin.backend.konan.library.SplitLibraryReader
import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.copyTo
@@ -89,8 +88,8 @@ class Library(val name: String, val requestedRepository: String?, val target: St
manifestFile.bufferedReader().use { reader ->
header.load(reader)
}
val headerAbiVersion = header.getProperty("abi_version")!!
val moduleName = header.getProperty("module_name")!!
val headerAbiVersion = library.abiVersion
val moduleName = ModuleDeserializer(library.moduleHeaderData).moduleName
println("")
println("Resolved to: ${library.libDir.absolutePath}")
@@ -129,9 +128,8 @@ class Library(val name: String, val requestedRepository: String?, val target: St
fun contents() {
val library = libraryInRepoOrCurrentDir(repository, name)
val moduleName = library.moduleName
val printer = PrettyPrinter(
library.tableOfContents, {name -> library.packageMetadata(name)})
library.moduleHeaderData, {name -> library.packageMetadata(name)})
printer.packageFragmentNameList.forEach{
printer.printPackageFragment(it)
@@ -143,19 +141,19 @@ class Library(val name: String, val requestedRepository: String?, val target: St
val currentAbiVersion = 1
val File.konanLibrary
get() = SplitLibraryReader(this, currentAbiVersion, null)
get() = LibraryReaderImpl(this, currentAbiVersion, null)
fun libraryInRepo(repository: File, name: String): SplitLibraryReader {
fun libraryInRepo(repository: File, name: String): LibraryReaderImpl {
val resolver = KonanLibrarySearchPathResolver(listOf(repository.absolutePath), null, null, skipCurrentDir = true)
return resolver.resolve(name).konanLibrary
}
fun libraryInCurrentDir(name: String): SplitLibraryReader {
fun libraryInCurrentDir(name: String): LibraryReaderImpl {
val resolver = KonanLibrarySearchPathResolver(emptyList(), null, null)
return resolver.resolve(name).konanLibrary
}
fun libraryInRepoOrCurrentDir(repository: File, name: String): SplitLibraryReader {
fun libraryInRepoOrCurrentDir(repository: File, name: String): LibraryReaderImpl {
val resolver = KonanLibrarySearchPathResolver(listOf(repository.absolutePath), null, null)
return resolver.resolve(name).konanLibrary
}