Splitted library writer into kotlin and konan parts

Moved to util-io and util-klib for klib code commonization
Moved the common writer to Kotlin workspace

backend.native/build.gradle

@@ -113,6 +113,8 @@ configurations {
     kotlin_reflect_jar
     kotlin_script_runtime_jar
     kotlin_native_utils_jar
+    kotlin_util_io_jar
+    kotlin_util_klib_jar
     trove4j_jar
     kotlinCommonSources
 
@@ -133,6 +135,8 @@ dependencies {
     kotlin_reflect_jar "$kotlinReflectModule@jar"
     kotlin_script_runtime_jar "$kotlinScriptRuntimeModule@jar"
     kotlin_native_utils_jar "$kotlinNativeUtilsModule@jar"
+    kotlin_util_io_jar "$kotlinUtilIoModule@jar"
+    kotlin_util_klib_jar "$kotlinUtilKlibModule@jar"
 
     [kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each {
         kotlinCommonSources(it) { transitive = false }
@@ -234,7 +238,7 @@ jar {
     dependsOn ':runtime:hostRuntime', 'external_jars'
 }
 
-def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils']
+def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils', 'util_io', 'util_klib']
 
 task trove4jCopy(type: Copy) {
     from configurations.getByName("trove4j_jar") {
@@ -285,4 +289,4 @@ task debugCompiler(type: JavaExec) {
             "-Djava.library.path=${distDir.canonicalPath}/konan/nativelib"
     enableAssertions = true
     args = findProperty("konan.debug.args").toString().tokenize() ?: []
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt

@@ -11,10 +11,10 @@ import org.jetbrains.kotlin.backend.konan.library.impl.buildLibrary
 import org.jetbrains.kotlin.backend.konan.llvm.*
 import org.jetbrains.kotlin.backend.konan.llvm.Llvm
 import org.jetbrains.kotlin.konan.CURRENT
-import org.jetbrains.kotlin.konan.KonanAbiVersion
+import org.jetbrains.kotlin.library.KotlinAbiVersion
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.file.isBitcode
-import org.jetbrains.kotlin.konan.library.KonanLibraryVersioning
+import org.jetbrains.kotlin.library.*
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
 
@@ -89,10 +89,14 @@ internal fun produceOutput(context: Context) {
             val output = context.config.outputFiles.outputName
             val libraryName = context.config.moduleId
             val neededLibraries = context.librariesWithDependencies
-            val abiVersion = KonanAbiVersion.CURRENT
+            val abiVersion = KotlinAbiVersion.CURRENT
             val compilerVersion = KonanVersion.CURRENT
             val libraryVersion = config.get(KonanConfigKeys.LIBRARY_VERSION)
-            val versions = KonanLibraryVersioning(abiVersion = abiVersion, libraryVersion = libraryVersion, compilerVersion = compilerVersion)
+            val versions = KonanLibraryVersioning(
+                abiVersion = abiVersion,
+                libraryVersion = libraryVersion,
+                compilerVersion = compilerVersion
+            )
             val target = context.config.target
             val nopack = config.getBoolean(KonanConfigKeys.NOPACK)
             val manifestProperties = context.config.manifestProperties
@@ -101,7 +105,10 @@ internal fun produceOutput(context: Context) {
                 context.config.nativeLibraries,
                 context.config.includeBinaries,
                 neededLibraries,
-                context.serializedLinkData!!,
+                context.serializedMetadata!!,
+                // During serializer and library commonizations we've got duplicate SerializedIr in
+                // two different packages. TODO: Eliminate them eventually.
+                SerializedIr(context.serializedIr!!.module, context.serializedIr!!.combinedDeclarationFilePath),
                 versions,
                 target,
                 output,
@@ -148,4 +155,4 @@ private fun embedAppleLinkerOptionsToBitcode(llvm: Llvm, config: KonanConfig) {
             llvm.nativeDependenciesToLink.flatMap { findEmbeddableOptions(it.linkerOpts) }
 
     embedLlvmLinkOptions(llvm.llvmModule, optionsToEmbed)
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -13,7 +13,8 @@ import org.jetbrains.kotlin.backend.common.descriptors.WrappedTypeParameterDescr
 import org.jetbrains.kotlin.backend.common.validateIrModule
 import org.jetbrains.kotlin.backend.konan.descriptors.*
 import org.jetbrains.kotlin.backend.konan.ir.KonanIr
-import org.jetbrains.kotlin.backend.konan.library.LinkData
+import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
+import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.backend.konan.llvm.*
 import org.jetbrains.kotlin.backend.konan.lower.DECLARATION_ORIGIN_BRIDGE_METHOD
 import org.jetbrains.kotlin.backend.konan.optimizations.Devirtualization
@@ -49,10 +50,12 @@ import kotlin.LazyThreadSafetyMode.PUBLICATION
 import kotlin.reflect.KProperty
 import org.jetbrains.kotlin.backend.common.ir.copyTo
 import org.jetbrains.kotlin.backend.common.serialization.KotlinMangler
+import org.jetbrains.kotlin.backend.common.serialization.SerializedIr
 import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport
 import org.jetbrains.kotlin.backend.konan.llvm.coverage.CoverageManager
 import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterSymbolImpl
 import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 
 /**
  * Offset for synthetic elements created by lowerings and not attributable to other places in the source code.
@@ -264,11 +267,11 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
         ClassLayoutBuilder(irClass, this)
     }
 
-    // We serialize untouched descriptor tree and inline IR bodies
-    // right after the frontend.
+    // We serialize untouched descriptor tree and IR.
     // But we have to wait until the code generation phase,
     // to dump this information into generated file.
-    var serializedLinkData: LinkData? = null
+    var serializedMetadata: SerializedMetadata? = null
+    var serializedIr: SerializedIr? = null
     var dataFlowGraph: ByteArray? = null
 
     val librariesWithDependencies by lazy {
@@ -322,6 +325,7 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
     val llvmImports: LlvmImports = Llvm.ImportsImpl(this)
     lateinit var llvmDeclarations: LlvmDeclarations
     lateinit var bitcodeFileName: String
+    lateinit var library: KonanLibraryLayout
 
     val cStubsManager = CStubsManager(config.target)
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -22,9 +22,10 @@ import org.jetbrains.kotlin.konan.library.libraryResolver
 import org.jetbrains.kotlin.konan.properties.loadProperties
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.library.resolver.TopologicalLibraryOrder
-import org.jetbrains.kotlin.konan.library.toUnresolvedLibraries
 import org.jetbrains.kotlin.konan.util.Logger
 import kotlin.system.exitProcess
+import org.jetbrains.kotlin.library.toUnresolvedLibraries
+import org.jetbrains.kotlin.konan.KonanVersion
 
 class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt

@@ -1,29 +1,26 @@
 package org.jetbrains.kotlin.backend.konan
 
-import llvm.LLVMWriteBitcodeToFile
 import org.jetbrains.kotlin.backend.common.LoggingContext
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
 import org.jetbrains.kotlin.backend.common.phaser.*
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
 import org.jetbrains.kotlin.backend.konan.descriptors.isForwardDeclarationModule
 import org.jetbrains.kotlin.backend.konan.descriptors.konanLibrary
 import org.jetbrains.kotlin.backend.konan.ir.KonanSymbols
 import org.jetbrains.kotlin.backend.konan.llvm.*
 import org.jetbrains.kotlin.backend.konan.lower.ExpectToActualDefaultValueCopier
 import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport
-import org.jetbrains.kotlin.backend.konan.serialization.*
+import org.jetbrains.kotlin.backend.konan.serialization.KonanDeclarationTable
+import org.jetbrains.kotlin.backend.konan.serialization.KonanIrLinker
+import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer
+import org.jetbrains.kotlin.backend.konan.serialization.KonanSerializationUtil
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
-import org.jetbrains.kotlin.backend.konan.library.impl.buildLibrary
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.SymbolTable
 import org.jetbrains.kotlin.ir.util.patchDeclarationParents
-import org.jetbrains.kotlin.konan.CURRENT
-import org.jetbrains.kotlin.konan.KonanAbiVersion
-import org.jetbrains.kotlin.konan.KonanVersion
-import org.jetbrains.kotlin.konan.library.KonanLibraryVersioning
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
 import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
@@ -181,10 +178,9 @@ internal val patchDeclarationParents0Phase = konanUnitPhase(
 internal val serializerPhase = konanUnitPhase(
         op = {
             val declarationTable = KonanDeclarationTable(irModule!!.irBuiltins, DescriptorTable())
-            val serializedIr = KonanIrModuleSerializer(this, declarationTable).serializedIrModule(irModule!!)
+            serializedIr = KonanIrModuleSerializer(this, declarationTable).serializedIrModule(irModule!!)
             val serializer = KonanSerializationUtil(this, config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, declarationTable)
-            serializedLinkData =
-                    serializer.serializeModule(moduleDescriptor, /*if (!config.isInteropStubs) serializedIr else null*/ serializedIr)
+            serializedMetadata = serializer.serializeModule(moduleDescriptor)
         },
         name = "Serializer",
         description = "Serialize descriptor tree and inline IR bodies"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt

@@ -6,31 +6,17 @@
 package org.jetbrains.kotlin.backend.konan.library
 
 import llvm.LLVMModuleRef
-import org.jetbrains.kotlin.backend.common.serialization.SerializedIr
-import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.konan.library.KonanLibraryVersioning
-import org.jetbrains.kotlin.konan.properties.Properties
+import org.jetbrains.kotlin.library.BaseWriter
+import org.jetbrains.kotlin.library.IrWriter
+import org.jetbrains.kotlin.library.MetadataWriter
 
-interface KonanLibraryWriter {
-    val versions: KonanLibraryVersioning
-    fun addLinkData(linkData: LinkData)
-    fun addNativeBitcode(library: String)
+interface TargetedWriter {
     fun addIncludedBinary(library: String)
+}
+
+interface BitcodeWriter : TargetedWriter {
     fun addKotlinBitcode(llvmModule: LLVMModuleRef)
-    fun addLinkDependencies(libraries: List<KonanLibrary>)
-    fun addManifestAddend(properties: Properties)
-    fun addDataFlowGraph(dataFlowGraph: ByteArray)
-    val mainBitcodeFileName: String
-    fun commit()
+    fun addNativeBitcode(library: String)
 }
 
-class LinkData(
-    val module: ByteArray,
-    val fragments: List<List<ByteArray>>,
-    val fragmentNames: List<String>,
-    val ir: SerializedIr? = null
-)
-
-interface MetadataWriter {
-    fun addLinkData(linkData: LinkData)
-}
+interface KonanLibraryWriter : MetadataWriter, BaseWriter, IrWriter, BitcodeWriter

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/BitcodeWriterImpl.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.kotlin.backend.konan.library.impl
+
+import llvm.LLVMModuleRef
+import llvm.LLVMWriteBitcodeToFile
+import org.jetbrains.kotlin.backend.konan.library.BitcodeWriter
+import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.library.BitcodeKotlinLibraryLayout
+import org.jetbrains.kotlin.konan.library.TargetedKotlinLibraryLayout
+
+open class TargetedWriterImpl(val targetLayout: TargetedKotlinLibraryLayout) {
+    init {
+        targetLayout.targetDir.mkdirs()
+        targetLayout.includedDir.mkdirs()
+    }
+
+    fun addIncludedBinary(library: String) {
+        val basename = File(library).name
+        File(library).copyTo(File(targetLayout.includedDir, basename))
+    }
+}
+
+class BitcodeWriterImpl(
+    libraryLayout: BitcodeKotlinLibraryLayout
+) : BitcodeWriter, TargetedWriterImpl(libraryLayout) {
+
+    val bitcodeLayout = libraryLayout
+
+    init {
+        bitcodeLayout.kotlinDir.mkdirs()
+        bitcodeLayout.nativeDir.mkdirs()
+    }
+
+    override fun addNativeBitcode(library: String) {
+        val basename = File(library).name
+        File(library).copyTo(File(bitcodeLayout.nativeDir, basename))
+    }
+
+    override fun addKotlinBitcode(llvmModule: LLVMModuleRef) {
+        //libraryLayout.llvmModule = llvmModule
+        LLVMWriteBitcodeToFile(llvmModule, bitcodeLayout.mainBitcodeFileName)
+    }
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt

@@ -6,130 +6,67 @@
 package org.jetbrains.kotlin.backend.konan.library.impl
 
 import llvm.LLVMModuleRef
-import llvm.LLVMWriteBitcodeToFile
+import org.jetbrains.kotlin.library.SerializedIr
+import org.jetbrains.kotlin.backend.konan.library.BitcodeWriter
 import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
-import org.jetbrains.kotlin.backend.konan.library.LinkData
-import org.jetbrains.kotlin.konan.file.*
-import org.jetbrains.kotlin.konan.library.*
-import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION
-import org.jetbrains.kotlin.konan.properties.*
+import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
+import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.library.*
+import org.jetbrains.kotlin.library.impl.BaseWriterImpl
+import org.jetbrains.kotlin.library.impl.IrWriterImpl
+import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
+import org.jetbrains.kotlin.library.impl.MetadataWriterImpl
+
+class KonanLibraryLayoutForWriter(
+    override val libDir: File,
+    override val target: KonanTarget
+) : KonanLibraryLayout, KotlinLibraryLayoutForWriter(libDir)
 
-abstract class FileBasedLibraryWriter (val file: File): KonanLibraryWriter
 
 /**
  * Requires non-null [target].
  */
-class LibraryWriterImpl(
-        override val libDir: File,
-        moduleName: String,
-        override val versions: KonanLibraryVersioning,
-        override val target: KonanTarget,
-        val nopack: Boolean = false
-): FileBasedLibraryWriter(libDir), KonanLibraryLayout {
+class KonanLibraryWriterImpl(
+    libDir: File,
+    moduleName: String,
+    versions: KonanLibraryVersioning,
+    target: KonanTarget,
+    nopack: Boolean = false,
 
-    constructor(path: String, moduleName: String, versions: KonanLibraryVersioning, target: KonanTarget, nopack: Boolean):
-        this(File(path), moduleName, versions, target, nopack)
+    val layout: KonanLibraryLayoutForWriter = KonanLibraryLayoutForWriter(libDir, target),
 
-    override val libraryName = libDir.path
-    val klibFile 
-       get() = File("${libDir.path}.$KLIB_FILE_EXTENSION")
+    base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack),
+    bitcode: BitcodeWriter = BitcodeWriterImpl(layout),
+    metadata: MetadataWriter = MetadataWriterImpl(layout),
+    ir: IrWriter = IrWriterImpl(layout)
 
-    // TODO: Experiment with separate bitcode files.
-    // Per package or per class.
-    val mainBitcodeFile = File(kotlinDir, "program.kt.bc")
-    override val mainBitcodeFileName = mainBitcodeFile.path
-    val manifestProperties = Properties()
-
-    init {
-        // TODO: figure out the proper policy here.
-        libDir.deleteRecursively()
-        klibFile.delete()
-        libDir.mkdirs()
-        linkdataDir.mkdirs()
-        targetDir.mkdirs()
-        kotlinDir.mkdirs()
-        nativeDir.mkdirs()
-        includedDir.mkdirs()
-        resourcesDir.mkdirs()
-        irDir.mkdirs()
-        // TODO: <name>:<hash> will go somewhere around here.
-        manifestProperties.setProperty(KLIB_PROPERTY_UNIQUE_NAME, moduleName)
-        manifestProperties.writeKonanLibraryVersioning(versions)
-    }
-
-    var llvmModule: LLVMModuleRef? = null
-
-    override fun addKotlinBitcode(llvmModule: LLVMModuleRef) {
-        this.llvmModule = llvmModule
-        LLVMWriteBitcodeToFile(llvmModule, mainBitcodeFileName)
-    }
-
-    override fun addLinkData(linkData: LinkData) {
-        MetadataWriterImpl(this).addLinkData(linkData)
-    }
-
-    override fun addNativeBitcode(library: String) {
-        val basename = File(library).name
-        File(library).copyTo(File(nativeDir, basename)) 
-    }
-
-    override fun addIncludedBinary(library: String) {
-        val basename = File(library).name
-        File(library).copyTo(File(includedDir, basename)) 
-    }
-
-    override fun addLinkDependencies(libraries: List<KonanLibrary>) {
-        if (libraries.isEmpty()) {
-            manifestProperties.remove(KLIB_PROPERTY_DEPENDS)
-            // make sure there are no leftovers from the .def file.
-            return
-        } else {
-            val newValue = libraries.joinToString(" ") { it.uniqueName }
-            manifestProperties.setProperty(KLIB_PROPERTY_DEPENDS, newValue)
-            libraries.forEach { it ->
-                if (it.versions.libraryVersion != null) {
-                    manifestProperties.setProperty("${KLIB_PROPERTY_DEPENDENCY_VERSION}_${it.uniqueName}", it.versions.libraryVersion)
-                }
-            }
-        }
-    }
-
-    override fun addManifestAddend(properties: Properties) {
-        manifestProperties.putAll(properties)
-    }
-
-    override fun addDataFlowGraph(dataFlowGraph: ByteArray) {
-        dataFlowGraphFile.writeBytes(dataFlowGraph)
-    }
-
-    override fun commit() {
-        manifestProperties.saveToFile(manifestFile)
-        if (!nopack) {
-            libDir.zipDirAs(klibFile)
-            libDir.deleteRecursively()
-        }
-    }
-}
+) : BaseWriter by base, BitcodeWriter by bitcode, MetadataWriter by metadata, IrWriter by ir, KonanLibraryWriter
 
 internal fun buildLibrary(
-        natives: List<String>,
-        included: List<String>,
-        linkDependencies: List<KonanLibrary>,
-        linkData: LinkData,
-        versions: KonanLibraryVersioning,
-        target: KonanTarget,
-        output: String,
-        moduleName: String,
-        llvmModule: LLVMModuleRef?,
-        nopack: Boolean,
-        manifestProperties: Properties?,
-        dataFlowGraph: ByteArray?): KonanLibraryWriter {
+    natives: List<String>,
+    included: List<String>,
+    linkDependencies: List<KonanLibrary>,
+    metadata: SerializedMetadata,
+    ir: SerializedIr,
+    versions: KonanLibraryVersioning,
+    target: KonanTarget,
+    output: String,
+    moduleName: String,
+    llvmModule: LLVMModuleRef?,
+    nopack: Boolean,
+    manifestProperties: Properties?,
+    dataFlowGraph: ByteArray?
+): KonanLibraryLayout {
 
-    val library = LibraryWriterImpl(File(output), moduleName, versions, target, nopack)
+    val library = KonanLibraryWriterImpl(File(output), moduleName, versions, target, nopack)
 
     llvmModule?.let { library.addKotlinBitcode(it) }
-    library.addLinkData(linkData)
+    library.addMetadata(metadata)
+    library.addIr(ir)
+
     natives.forEach {
         library.addNativeBitcode(it)
     }
@@ -141,6 +78,5 @@ internal fun buildLibrary(
     dataFlowGraph?.let { library.addDataFlowGraph(it) }
 
     library.commit()
-    return library
-}
-
+    return library.layout
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt

@@ -1,34 +0,0 @@
-/*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-package org.jetbrains.kotlin.backend.konan.library.impl
-
-import org.jetbrains.kotlin.backend.konan.library.LinkData
-import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
-
-internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibraryLayout by libraryLayout {
-
-    fun addLinkData(linkData: LinkData) {
-        moduleHeaderFile.writeBytes(linkData.module)
-        linkData.ir?.let { irHeader.writeBytes(it.module) }
-        linkData.fragments.forEachIndexed { index, it ->
-            val packageFqName = linkData.fragmentNames[index]
-            val shortName = packageFqName.substringAfterLast(".")
-            val dir = packageFragmentsDir(packageFqName)
-            dir.deleteRecursively()
-            dir.mkdirs()
-            val numCount = it.size.toString().length
-            fun withLeadingZeros(i: Int) = String.format("%0${numCount}d", i)
-            for ((i, fragment) in it.withIndex()) {
-                packageFragmentFile(packageFqName, "${withLeadingZeros(i)}_$shortName").writeBytes(fragment)
-            }
-        }
-        linkData.ir?.combinedDeclarationFilePath?.let {
-            // TODO: use Files.move.
-            File(it).copyTo(irFile)
-        }
-    }
-}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt

@@ -6,30 +6,30 @@
 package org.jetbrains.kotlin.backend.konan.llvm
 
 import llvm.LLVMTypeRef
-import org.jetbrains.kotlin.backend.common.serialization.KotlinMangler
 import org.jetbrains.kotlin.backend.common.serialization.KotlinManglerImpl
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.descriptors.externalSymbolOrThrow
 import org.jetbrains.kotlin.backend.konan.descriptors.getAnnotationValue
 import org.jetbrains.kotlin.backend.konan.descriptors.isAbstract
-import org.jetbrains.kotlin.backend.konan.ir.*
+import org.jetbrains.kotlin.backend.konan.ir.allParameters
+import org.jetbrains.kotlin.backend.konan.ir.getObjCMethodInfo
+import org.jetbrains.kotlin.backend.konan.ir.isObjCClassMethod
+import org.jetbrains.kotlin.backend.konan.ir.isUnit
 import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler.isExported
-import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PropertyAccessorDescriptor
-import org.jetbrains.kotlin.descriptors.Visibilities
-import org.jetbrains.kotlin.descriptors.Visibility
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
 import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.util.fqNameForIrSerialization
+import org.jetbrains.kotlin.ir.types.getClass
+import org.jetbrains.kotlin.ir.util.fqNameSafe
 import org.jetbrains.kotlin.ir.util.isSuspend
 import org.jetbrains.kotlin.ir.util.isVararg
-import org.jetbrains.kotlin.konan.library.uniqueName
 import org.jetbrains.kotlin.backend.konan.isInlinedNative
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.library.uniqueName
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
 
 
 // This file describes the ABI for Kotlin descriptors of exported declarations.

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/DebugUtils.kt

@@ -9,8 +9,6 @@ import kotlinx.cinterop.allocArrayOf
 import kotlinx.cinterop.memScoped
 import llvm.*
 import org.jetbrains.kotlin.backend.konan.*
-import org.jetbrains.kotlin.builtins.KotlinBuiltIns
-import org.jetbrains.kotlin.builtins.UnsignedType
 import org.jetbrains.kotlin.builtins.UnsignedTypes
 import org.jetbrains.kotlin.ir.SourceManager.FileEntry
 import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
@@ -20,7 +18,6 @@ import org.jetbrains.kotlin.js.descriptorUtils.getJetTypeFqName
 import org.jetbrains.kotlin.konan.CURRENT
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.resolve.descriptorUtil.classId
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.TypeUtils
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt

@@ -15,6 +15,10 @@ import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.konan.KonanAbiVersion
+import org.jetbrains.kotlin.ir.util.fqNameSafe
+import org.jetbrains.kotlin.ir.util.isAnnotationClass
+import org.jetbrains.kotlin.ir.util.isInterface
+import org.jetbrains.kotlin.library.KotlinAbiVersion
 import org.jetbrains.kotlin.name.FqName
 
 internal class RTTIGenerator(override val context: Context) : ContextUtils {
@@ -90,7 +94,7 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
 
                     extendedInfo,
 
-                    Int32(KonanAbiVersion.CURRENT.version),
+                    Int32(KotlinAbiVersion.CURRENT.version),
 
                     Int32(size),
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt

@@ -9,9 +9,9 @@ import org.jetbrains.kotlin.backend.common.serialization.SerializedIr
 import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.descriptors.isExpectMember
 import org.jetbrains.kotlin.backend.konan.descriptors.isSerializableExpectClass
-import org.jetbrains.kotlin.backend.konan.library.LinkData
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
@@ -207,7 +207,7 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
                 .build()
     }
 
-    internal fun serializeModule(moduleDescriptor: ModuleDescriptor, serializedIr: SerializedIr?): LinkData {
+    internal fun serializeModule(moduleDescriptor: ModuleDescriptor): SerializedMetadata {
         val libraryProto = KonanProtoBuf.LinkDataLibrary.newBuilder()
         libraryProto.moduleName = moduleDescriptor.name.asString()
         val fragments = mutableListOf<List<ByteArray>>()
@@ -233,7 +233,7 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
         }
 
         val libraryAsByteArray = libraryProto.build().toByteArray()
-        return LinkData(libraryAsByteArray, fragments, fragmentNames, serializedIr)
+        return SerializedMetadata(libraryAsByteArray, fragments, fragmentNames)
     }
 }
 

build.gradle

@@ -59,6 +59,8 @@ ext {
     kotlinReflectModule="org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}"
     kotlinScriptRuntimeModule="org.jetbrains.kotlin:kotlin-script-runtime:${kotlinVersion}"
     kotlinNativeUtilsModule="org.jetbrains.kotlin:kotlin-native-utils:${kotlinVersion}"
+    kotlinUtilIoModule="org.jetbrains.kotlin:kotlin-util-io:${kotlinVersion}"
+    kotlinUtilKlibModule="org.jetbrains.kotlin:kotlin-util-klib:${kotlinVersion}"
 
     konanVersionFull = KonanVersionGeneratedKt.getCurrentKonanVersion()
 

extracted/konan.serializer/src/org/jetbrains/kotlin/konan/library/resolver/KonanLibraryResolver.kt

@@ -2,7 +2,7 @@ package org.jetbrains.kotlin.konan.library.resolver
 
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.SearchPathResolverWithTarget
-import org.jetbrains.kotlin.konan.library.UnresolvedLibrary
+import org.jetbrains.kotlin.library.UnresolvedLibrary
 
 interface KonanLibraryResolver {
 
@@ -13,9 +13,9 @@ interface KonanLibraryResolver {
      * resolves libraries and evaluates dependencies between them.
      */
     fun resolveWithDependencies(
-            unresolvedLibraries: List<UnresolvedLibrary>,
-            noStdLib: Boolean = false,
-            noDefaultLibs: Boolean = false
+        unresolvedLibraries: List<UnresolvedLibrary>,
+        noStdLib: Boolean = false,
+        noDefaultLibs: Boolean = false
     ): KonanLibraryResolveResult
 }
 

extracted/konan.serializer/src/org/jetbrains/kotlin/konan/library/resolver/impl/KonanLibraryResolverImpl.kt

@@ -20,15 +20,17 @@ import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.*
 import org.jetbrains.kotlin.konan.library.resolver.*
 import org.jetbrains.kotlin.konan.util.WithLogger
+import org.jetbrains.kotlin.library.UnresolvedLibrary
+import org.jetbrains.kotlin.library.unresolvedDependencies
 
 internal class KonanLibraryResolverImpl(
         override val searchPathResolver: SearchPathResolverWithTarget
 ): KonanLibraryResolver, WithLogger by searchPathResolver {
 
     override fun resolveWithDependencies(
-            unresolvedLibraries: List<UnresolvedLibrary>,
-            noStdLib: Boolean,
-            noDefaultLibs: Boolean
+        unresolvedLibraries: List<UnresolvedLibrary>,
+        noStdLib: Boolean,
+        noDefaultLibs: Boolean
     ) = findLibraries(unresolvedLibraries, noStdLib, noDefaultLibs)
             .leaveDistinct()
             .resolveDependencies()

gradle.properties

@@ -27,3 +27,7 @@ konanVersion=1.3.50
 org.gradle.jvmargs='-Dfile.encoding=UTF-8'
 org.gradle.workers.max=4
 #kotlinProjectPath=/Users/jetbrains/IdeaProjects/kotlin
+#sharedProjectPath=/Users/jetbrains/IdeaProjects/kotlin-native-shared
+kotlinProjectPath=/Users/jetbrains/kotlin-native/kotlin
+#sharedProjectPath=/Users/jetbrains/kotlin-native/kotlin-native-shared/kotlin-native-shared
+

klib/src/main/kotlin/org/jetbrains/kotlin/cli/klib/main.kt

@@ -12,14 +12,12 @@ import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.descriptors.konan.isKonanStdlib
 import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION_WITH_DOT
-import org.jetbrains.kotlin.konan.library.createKonanLibrary
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION_WITH_DOT
 import org.jetbrains.kotlin.konan.library.resolverByName
 import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
-import org.jetbrains.kotlin.konan.KonanAbiVersion
-import org.jetbrains.kotlin.konan.library.unpackZippedKonanLibraryTo
+import org.jetbrains.kotlin.library.unpackZippedKonanLibraryTo
 import org.jetbrains.kotlin.konan.utils.KonanFactories.DefaultDeserializedDescriptorFactory
 import org.jetbrains.kotlin.konan.util.Logger
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf

settings.gradle

@@ -56,6 +56,8 @@ if (hasProperty("kotlinProjectPath")) {
             substitute module('org.jetbrains.kotlin:kotlin-compiler') with project(':include:kotlin-compiler')
             substitute module("org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion") with project(':include:kotlin-stdlib-common-sources')
             substitute module("org.jetbrains.kotlin:kotlin-stdlib-gen:$kotlinVersion") with project(':tools:kotlin-stdlib-gen')
+            substitute module("org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion") with project(':kotlin-util-io')
+            substitute module("org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion") with project(':kotlin-util-klib')
         }
     }
 }

shared/build.gradle

@@ -63,4 +63,6 @@ jar {
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
     compile "org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion"
+    compile "org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion"
+    compile "org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion"
 }

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibrary.kt

@@ -3,63 +3,30 @@ package org.jetbrains.kotlin.konan.library
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.properties.propertyList
+import org.jetbrains.kotlin.library.*
 
-const val KLIB_PROPERTY_ABI_VERSION = "abi_version"
-const val KLIB_PROPERTY_COMPILER_VERSION = "compiler_version"
-const val KLIB_PROPERTY_DEPENDENCY_VERSION = "dependency_version"
-const val KLIB_PROPERTY_LIBRARY_VERSION = "library_version"
-const val KLIB_PROPERTY_UNIQUE_NAME = "unique_name"
 const val KLIB_PROPERTY_LINKED_OPTS = "linkerOpts"
-const val KLIB_PROPERTY_DEPENDS = "depends"
 const val KLIB_PROPERTY_INTEROP = "interop"
-const val KLIB_PROPERTY_PACKAGE = "package"
 const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations"
 const val KLIB_PROPERTY_INCLUDED_HEADERS = "includedHeaders"
 
-/**
- * An abstraction for getting access to the information stored inside of Kotlin/Native library.
- */
-interface KonanLibrary {
-
-    val libraryName: String
-    val libraryFile: File
-
-    // Whether this library is default (provided by Kotlin/Native distribution)?
-    val isDefault: Boolean
-
-    // Properties:
-    val manifestProperties: Properties
-    val linkerOpts: List<String>
-
-    // Paths:
-    val bitcodePaths: List<String>
-    val includedPaths: List<String>
-
+interface TargetedLibrary {
     val targetList: List<String>
-    val versions: KonanLibraryVersioning
-
-    val dataFlowGraph: ByteArray?
-    val moduleHeaderData: ByteArray
-    fun packageMetadataParts(fqName: String): Set<String>
-    fun packageMetadata(fqName: String, partName: String): ByteArray
-
-    val irHeader: ByteArray?
-    fun irDeclaration(index: Long, isLocal: Boolean): ByteArray
+    val includedPaths: List<String>
 }
 
-val KonanLibrary.uniqueName
-    get() = manifestProperties.getProperty(KLIB_PROPERTY_UNIQUE_NAME)!!
+interface BitcodeLibrary : TargetedLibrary {
+    val bitcodePaths: List<String>
+}
 
-val KonanLibrary.unresolvedDependencies: List<UnresolvedLibrary>
-    get() = manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true)
-            .map {
-                UnresolvedLibrary(it, manifestProperties.getProperty("dependency_version_$it"))
-            }
+interface KonanLibrary : BitcodeLibrary, KotlinLibrary {
+    val linkerOpts: List<String>
+}
 
 val KonanLibrary.isInterop
     get() = manifestProperties.getProperty(KLIB_PROPERTY_INTEROP) == "true"
 
-val KonanLibrary.packageFqName
+val KonanLibrary.packageFqName: String?
     get() = manifestProperties.getProperty(KLIB_PROPERTY_PACKAGE)
 
 val KonanLibrary.exportForwardDeclarations

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt

@@ -18,55 +18,34 @@ package org.jetbrains.kotlin.konan.library
 
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.library.IrKotlinLibraryLayout
+import org.jetbrains.kotlin.library.KotlinLibraryLayout
+import org.jetbrains.kotlin.library.MetadataKotlinLibraryLayout
 
-/**
- * This scheme describes the Kotlin/Native Library (KLIB) layout.
- */
-interface KonanLibraryLayout {
 
-    val libraryName: String
-
-    val libDir: File
+interface TargetedKotlinLibraryLayout : KotlinLibraryLayout {
     val target: KonanTarget?
         // This is a default implementation. Can't make it an assignment.
         get() = null
-
-    val manifestFile
-        get() = File(libDir, "manifest")
-    val resourcesDir
-        get() = File(libDir, "resources")
-
     val targetsDir
         get() = File(libDir, "targets")
     val targetDir
         get() = File(targetsDir, target!!.visibleName)
+    val includedDir
+        get() = File(targetDir, "included")
+}
 
+interface BitcodeKotlinLibraryLayout : TargetedKotlinLibraryLayout, KotlinLibraryLayout {
     val kotlinDir
         get() = File(targetDir, "kotlin")
     val nativeDir
         get() = File(targetDir, "native")
-    val includedDir
-        get() = File(targetDir, "included")
-
-    val linkdataDir
-        get() = File(libDir, "linkdata")
-    val moduleHeaderFile
-        get() = File(linkdataDir, "module")
-    val dataFlowGraphFile
-        get() = File(linkdataDir, "module_data_flow_graph")
-
-    fun packageFragmentsDir(packageName: String)
-            = File(linkdataDir, if (packageName == "") "root_package" else "package_$packageName")
-
-    fun packageFragmentFile(packageFqName: String, partName: String) =
-            File(packageFragmentsDir(packageFqName), "$partName$KLIB_METADATA_FILE_EXTENSION_WITH_DOT")
-    val irDir
-        get() = File(libDir, "ir")
-    val irFile
-        get() = File(irDir, "irCombined.knd")
-    val irHeader
-        get() = File(irDir, "irHeaders.kni")
-    val irIndex: File
-        get() = File(irDir, "uniqIdTableDump.txt")
-
+    // TODO: Experiment with separate bitcode files.
+    // Per package or per class.
+    val mainBitcodeFile
+        get() = File(kotlinDir, "program.kt.bc")
+    val mainBitcodeFileName
+        get() = mainBitcodeFile.path
 }
+
+interface KonanLibraryLayout : MetadataKotlinLibraryLayout, BitcodeKotlinLibraryLayout, IrKotlinLibraryLayout

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryUtils.kt

@@ -1,49 +0,0 @@
-package org.jetbrains.kotlin.konan.library
-
-import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.file.file
-import org.jetbrains.kotlin.konan.file.withMutableZipFileSystem
-import org.jetbrains.kotlin.konan.library.impl.DefaultMetadataReaderImpl
-import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
-import org.jetbrains.kotlin.konan.library.impl.zippedKonanLibraryChecks
-import org.jetbrains.kotlin.konan.target.KonanTarget
-
-const val KLIB_FILE_EXTENSION = "klib"
-const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION"
-
-const val KLIB_METADATA_FILE_EXTENSION = "knm"
-const val KLIB_METADATA_FILE_EXTENSION_WITH_DOT = ".$KLIB_METADATA_FILE_EXTENSION"
-
-fun File.unpackZippedKonanLibraryTo(newDir: File) {
-
-    // First, run validity checks for the given KLIB file.
-    zippedKonanLibraryChecks(this)
-
-    if (newDir.exists) {
-        if (newDir.isDirectory)
-            newDir.deleteRecursively()
-        else
-            newDir.delete()
-    }
-
-    this.withMutableZipFileSystem {
-        it.file("/").recursiveCopyTo(newDir)
-    }
-    check(newDir.exists) { "Could not unpack $this as $newDir." }
-}
-
-val List<String>.toUnresolvedLibraries
-    get() = this.map {
-
-        val version = it.substringAfterLast('@', "")
-                    .let { if (it.isEmpty()) null else it}
-        val name = it.substringBeforeLast('@')
-        UnresolvedLibrary(name, version)
-    }
-
-fun createKonanLibrary(
-        libraryFile: File,
-        target: KonanTarget? = null,
-        isDefault: Boolean = false,
-        metadataReader: MetadataReader = DefaultMetadataReaderImpl
-): KonanLibrary = KonanLibraryImpl(libraryFile, target, isDefault, metadataReader)

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryVersioning.kt

@@ -1,28 +0,0 @@
-package org.jetbrains.kotlin.konan.library
-
-import org.jetbrains.kotlin.konan.KonanVersion
-import org.jetbrains.kotlin.konan.parseKonanVersion
-import org.jetbrains.kotlin.konan.KonanAbiVersion
-import org.jetbrains.kotlin.konan.properties.Properties
-import org.jetbrains.kotlin.konan.parseKonanAbiVersion
-
-
-data class KonanLibraryVersioning(
-    val libraryVersion: String?,
-    val compilerVersion: KonanVersion?,
-    val abiVersion: KonanAbiVersion?
-)
-
-fun Properties.writeKonanLibraryVersioning(versions: KonanLibraryVersioning) {
-    versions.abiVersion ?. let { this.setProperty(KLIB_PROPERTY_ABI_VERSION, it.toString()) }
-    versions.libraryVersion ?. let { this.setProperty(KLIB_PROPERTY_LIBRARY_VERSION, it) }
-    versions.compilerVersion ?. let { this.setProperty(KLIB_PROPERTY_COMPILER_VERSION, "${versions.compilerVersion.toString(true, true)}") }
-}
-
-fun Properties.readKonanLibraryVersioning(): KonanLibraryVersioning {
-    val abiVersion = this.getProperty(KLIB_PROPERTY_ABI_VERSION)?.parseKonanAbiVersion()
-    val libraryVersion = this.getProperty(KLIB_PROPERTY_LIBRARY_VERSION)
-    val compilerVersion = this.getProperty(KLIB_PROPERTY_COMPILER_VERSION)?.parseKonanVersion()
-
-    return KonanLibraryVersioning(abiVersion = abiVersion, libraryVersion = libraryVersion, compilerVersion = compilerVersion)
-}

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt

@@ -1,6 +0,0 @@
-package org.jetbrains.kotlin.konan.library
-
-interface MetadataReader {
-    fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
-    fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray
-}

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt

@@ -1,13 +1,17 @@
 package org.jetbrains.kotlin.konan.library
 
-import org.jetbrains.kotlin.konan.KonanAbiVersion
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.impl.KonanLibraryImpl
+import org.jetbrains.kotlin.konan.library.impl.createKonanLibrary
 import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.*
 import kotlin.system.exitProcess
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION_WITH_DOT
+import org.jetbrains.kotlin.library.KotlinAbiVersion
+import org.jetbrains.kotlin.library.UnresolvedLibrary
+import org.jetbrains.kotlin.library.uniqueName
 
 // FIXME(ddol): KLIB-REFACTORING-CLEANUP: remove the constants below:
 const val KONAN_STDLIB_NAME = "stdlib"
@@ -24,7 +28,7 @@ interface SearchPathResolver : WithLogger {
 // FIXME(ddol): KLIB-REFACTORING-CLEANUP: remove this interface!
 interface SearchPathResolverWithTarget: SearchPathResolver {
     val target: KonanTarget
-    val knownAbiVersions: List<KonanAbiVersion>?
+    val knownAbiVersions: List<KotlinAbiVersion>?
     val knownCompilerVersions: List<KonanVersion>?
 }
 
@@ -47,7 +51,7 @@ fun defaultResolver(
         repositories,
         directLibs,
         target,
-        listOf(KonanAbiVersion.CURRENT),
+        listOf(KotlinAbiVersion.CURRENT),
         compatibleCompilerVersions,
         distribution.klib,
         distribution.localKonanDir.absolutePath,
@@ -179,7 +183,7 @@ internal class KonanLibraryProperResolver(
     repositories: List<String>,
     directLibs: List<String>,
     override val target: KonanTarget,
-    override val knownAbiVersions: List<KonanAbiVersion>?,
+    override val knownAbiVersions: List<KotlinAbiVersion>?,
     override val knownCompilerVersions: List<KonanVersion>?,
     distributionKlib: String?,
     localKonanDir: String?,

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/UnresolvedLibrary.kt

@@ -1,10 +0,0 @@
-package org.jetbrains.kotlin.konan.library
-
-data class UnresolvedLibrary(
-    val path: String,
-    val libraryVersion: String?) {
-
-    fun substitutePath(newPath: String): UnresolvedLibrary {
-        return UnresolvedLibrary(newPath, libraryVersion)
-    }
-}

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt

@@ -1,94 +0,0 @@
-/**
- * Copyright 2010-2019 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.konan.library.impl
-
-import org.jetbrains.kotlin.konan.file.File
-import java.io.RandomAccessFile
-import java.nio.channels.FileChannel
-
-data class DeclarationId(val id: Long, val isLocal: Boolean)
-
-class CombinedIrFileReader(file: File) {
-    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
-    private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
-
-    init {
-        val declarationsCount = buffer.int
-        for (i in 0 until declarationsCount) {
-            val id = buffer.long
-            val isLocal = buffer.int != 0
-            val offset = buffer.int
-            val size = buffer.int
-            declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
-        }
-    }
-
-    fun declarationBytes(id: DeclarationId): ByteArray {
-        val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
-        val result = ByteArray(offsetSize.second)
-        buffer.position(offsetSize.first)
-        buffer.get(result, 0, offsetSize.second)
-        return result
-    }
-}
-
-private const val SINGLE_INDEX_RECORD_SIZE = 20  // sizeof(Long) + 3 * sizeof(Int).
-private const val INDEX_HEADER_SIZE = 4  // sizeof(Int).
-
-class CombinedIrFileWriter(val declarationCount: Int) {
-    private var currentDeclaration = 0
-    private var currentPosition = 0
-    private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
-    private val randomAccessFile = RandomAccessFile(file.path, "rw")
-
-    init {
-        randomAccessFile.writeInt(declarationCount)
-        assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
-        for (i in 0 until declarationCount) {
-            randomAccessFile.writeLong(-1) // id
-            randomAccessFile.writeInt(-1)  // isLocal
-            randomAccessFile.writeInt(-1)  // offset
-            randomAccessFile.writeInt(-1)  // size
-        }
-        currentPosition = randomAccessFile.filePointer.toInt()
-        assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
-    }
-
-    fun skipDeclaration() {
-        currentDeclaration++
-    }
-
-    fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
-        randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
-        randomAccessFile.writeLong(id.id)
-        randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
-        randomAccessFile.writeInt(currentPosition)
-        randomAccessFile.writeInt(bytes.size)
-        randomAccessFile.seek(currentPosition.toLong())
-        randomAccessFile.write(bytes)
-        assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
-        currentPosition = randomAccessFile.filePointer.toInt()
-        currentDeclaration++
-    }
-
-    fun finishWriting(): File {
-        assert(currentDeclaration == declarationCount)
-        randomAccessFile.close()
-        return file
-    }
-}
-

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt

@@ -1,14 +0,0 @@
-package org.jetbrains.kotlin.konan.library.impl
-
-import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
-import org.jetbrains.kotlin.konan.library.MetadataReader
-
-internal object DefaultMetadataReaderImpl : MetadataReader {
-
-    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray =
-            libraryLayout.moduleHeaderFile.readBytes()
-
-    override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray =
-            libraryLayout.packageFragmentFile(fqName, partName).readBytes()
-
-}

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt

@@ -24,82 +24,80 @@ import org.jetbrains.kotlin.konan.properties.propertyList
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
 import org.jetbrains.kotlin.konan.util.substitute
+import org.jetbrains.kotlin.library.*
+import org.jetbrains.kotlin.library.impl.*
 
-class KonanLibraryImpl(
-        override val libraryFile: File,
-        internal val target: KonanTarget?,
-        override val isDefault: Boolean,
-        private val metadataReader: MetadataReader
-) : KonanLibrary {
+open class TargetedLibraryImpl(
+    private val access: TargetedLibraryAccess<TargetedKotlinLibraryLayout>,
+    private val base: BaseKotlinLibrary
+) : TargetedLibrary, BaseKotlinLibrary by base {
 
-    // For the zipped libraries inPlace gives files from zip file system
-    // whereas realFiles extracts them to /tmp.
-    // For unzipped libraries inPlace and realFiles are the same
-    // providing files in the library directory.
+    private val target: KonanTarget? get() = access.target
 
-    private val layout = createKonanLibraryLayout(libraryFile, target)
-
-    override val libraryName: String by lazy { layout.inPlace { it.libraryName } }
+    override val targetList by lazy {
+        access.inPlace { it: TargetedKotlinLibraryLayout ->
+            it.targetsDir.listFiles.map {
+                it.name
+            }
+        }
+    }
 
     override val manifestProperties: Properties by lazy {
-        val properties = layout.inPlace { it.manifestFile.loadProperties() }
-        if (target != null) substitute(properties, defaultTargetSubstitutions(target))
+        val properties = access.inPlace {
+            it.manifestFile.loadProperties()
+        }
+        target?.let { substitute(properties, defaultTargetSubstitutions(it)) }
         properties
     }
 
-    override val versions: KonanLibraryVersioning
-        get() = manifestProperties.readKonanLibraryVersioning()
+    override val includedPaths: List<String>
+        get() = access.realFiles {
+            it.includedDir.listFilesOrEmpty.map { it.absolutePath }
+        }
+}
+
+open class BitcodeLibraryImpl(
+    private val access: BitcodeLibraryAccess<BitcodeKotlinLibraryLayout>,
+    targeted: TargetedLibrary
+) : BitcodeLibrary, TargetedLibrary by targeted {
+    override val bitcodePaths: List<String>
+        get() = access.realFiles { it: BitcodeKotlinLibraryLayout ->
+            (it.kotlinDir.listFilesOrEmpty + it.nativeDir.listFilesOrEmpty).map { it.absolutePath }
+        }
+}
+
+class KonanLibraryImpl(
+    targeted: TargetedLibraryImpl,
+    metadata: MetadataLibraryImpl,
+    ir: IrLibraryImpl,
+    bitcode: BitcodeLibraryImpl
+) : KonanLibrary,
+    BaseKotlinLibrary by targeted,
+    MetadataLibrary by metadata,
+    IrLibrary by ir,
+    BitcodeLibrary by bitcode {
 
     override val linkerOpts: List<String>
         get() = manifestProperties.propertyList(KLIB_PROPERTY_LINKED_OPTS, escapeInQuotes = true)
-
-    override val bitcodePaths: List<String>
-        get() = layout.realFiles { (it.kotlinDir.listFilesOrEmpty + it.nativeDir.listFilesOrEmpty).map { it.absolutePath } }
-
-    override val includedPaths: List<String>
-        get() = layout.realFiles { it.includedDir.listFilesOrEmpty.map { it.absolutePath } }
-
-    override val targetList by lazy { layout.inPlace { it.targetsDir.listFiles.map { it.name } } }
-
-    override val dataFlowGraph by lazy { layout.inPlace { it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } } }
-
-    override val moduleHeaderData: ByteArray by lazy { layout.inPlace { metadataReader.loadSerializedModule(it) } }
-
-    override fun packageMetadata(packageFqName: String, partName: String) =
-            layout.inPlace { metadataReader.loadSerializedPackageFragment(it, packageFqName, partName) }
-
-    override fun packageMetadataParts(fqName: String): Set<String> =
-            layout.inPlace { inPlaceLayout ->
-                val fileList =
-                        inPlaceLayout.packageFragmentsDir(fqName)
-                                .listFiles
-                                .mapNotNull {
-                                    it.name
-                                            .substringBeforeLast(KLIB_METADATA_FILE_EXTENSION_WITH_DOT, missingDelimiterValue = "")
-                                            .takeIf { it.isNotEmpty() }
-                                }
-
-                fileList.toSortedSet().also {
-                    require(it.size == fileList.size) { "Duplicated names: ${fileList.groupingBy { it }.eachCount().filter { (_, count) -> count > 1 }}" }
-                }
-            }
-
-    override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let {
-        if (it.exists) loadIrHeader() else null }
-    }}
-
-    override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal)
-
-    override fun toString() = "$libraryName[default=$isDefault]"
-
-    private val combinedDeclarations: CombinedIrFileReader by lazy {
-        CombinedIrFileReader(layout.realFiles {
-            it.irFile
-        })
-    }
-    private fun loadIrHeader(): ByteArray =
-            layout.inPlace {  it.irHeader.readBytes() }
-
-    private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
-            combinedDeclarations.declarationBytes(DeclarationId(index, isLocal))
 }
+
+
+fun createKonanLibrary(
+    libraryFile: File,
+    target: KonanTarget? = null,
+    isDefault: Boolean = false
+): KonanLibrary {
+    val baseAccess = BaseLibraryAccess<KotlinLibraryLayout>(libraryFile)
+    val targetedAccess = TargetedLibraryAccess<TargetedKotlinLibraryLayout>(libraryFile, target)
+    val metadataAccess = MetadataLibraryAccess<MetadataKotlinLibraryLayout>(libraryFile)
+    val irAccess = IrLibraryAccess<IrKotlinLibraryLayout>(libraryFile)
+    val bitcodeAccess = BitcodeLibraryAccess<BitcodeKotlinLibraryLayout>(libraryFile, target)
+
+    val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
+    val targeted = TargetedLibraryImpl(targetedAccess, base)
+    val metadata = MetadataLibraryImpl(metadataAccess)
+    val ir = IrLibraryImpl(irAccess)
+    val bitcode = BitcodeLibraryImpl(bitcodeAccess, targeted)
+
+    return KonanLibraryImpl(targeted, metadata, ir, bitcode)
+}

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt

@@ -1,93 +1,61 @@
 package org.jetbrains.kotlin.konan.library.impl
 
-import org.jetbrains.kotlin.konan.file.*
-import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION
-import org.jetbrains.kotlin.konan.library.KLIB_FILE_EXTENSION_WITH_DOT
-import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
+import org.jetbrains.kotlin.konan.file.File
+import org.jetbrains.kotlin.konan.library.*
 import org.jetbrains.kotlin.konan.target.KonanTarget
-import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
+import org.jetbrains.kotlin.library.*
+import org.jetbrains.kotlin.library.impl.*
 import java.nio.file.FileSystem
 
-interface KonanLibraryLayoutImpl: KonanLibraryLayout {
-    fun <T> inPlace(action: (KonanLibraryLayout) -> T): T
-    fun <T> realFiles (action: (KonanLibraryLayout) -> T): T
-}
+open class TargetedLibraryLayoutImpl(klib: File, override val target: KonanTarget?) :
+    KotlinLibraryLayoutImpl(klib), TargetedKotlinLibraryLayout {
 
-private class ZippedKonanLibraryLayout(val klibFile: File, override val target: KonanTarget?): KonanLibraryLayoutImpl {
-
-    init { zippedKonanLibraryChecks(klibFile) }
-
-    override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
-
-    override val libDir = File("/")
-
-    override fun <T> realFiles(action: (KonanLibraryLayout) -> T): T {
-        return action(FileExtractor(this))!!
+    override val extractingToTemp: TargetedKotlinLibraryLayout by lazy {
+        ExtractingTargetedLibraryImpl(this)
     }
 
-    override fun <T> inPlace(action: (KonanLibraryLayout) -> T): T {
-        return klibFile.withZipFileSystem { zipFileSystem ->
-            action(DirectFromZip(this, zipFileSystem))
-        }
-    }
+    override fun directlyFromZip(zipFileSystem: FileSystem): TargetedKotlinLibraryLayout =
+        FromZipTargetedLibraryImpl(this, zipFileSystem)
+
 }
 
-internal fun zippedKonanLibraryChecks(klibFile: File) {
-    check(klibFile.exists) { "Could not find $klibFile." }
-    check(klibFile.isFile) { "Expected $klibFile to be a regular file." }
-
-    val extension = klibFile.extension
-    check(extension.isEmpty() || extension == KLIB_FILE_EXTENSION) {
-        "KLIB path has unexpected extension: $klibFile"
-    }
-}
-
-private class UnzippedKonanLibraryLayout(override val libDir: File, override val target: KonanTarget?): KonanLibraryLayoutImpl {
-    override val libraryName = libDir.path
-
-    override fun <T> inPlace(action: (KonanLibraryLayout) -> T): T = action(this)
-    override fun <T> realFiles (action: (KonanLibraryLayout) -> T): T = inPlace(action)
-}
-
-private class DirectFromZip(zippedLayout: ZippedKonanLibraryLayout, val zipFileSystem: FileSystem): KonanLibraryLayout {
-    override val libraryName = zippedLayout.libraryName
-    override val libDir = zipFileSystem.file(zippedLayout.libDir)
-}
-
-/**
- * This class automatically extracts pieces of the library on first access. Use it if you need
- * to pass extracted files to an external tool. Otherwise, stick to [DirectFromZip].
- */
-private class FileExtractor(val zippedLibraryLayout: ZippedKonanLibraryLayout): KonanLibraryLayout by zippedLibraryLayout {
-
-    override val manifestFile: File by lazy { extract(super.manifestFile) }
-
-    override val resourcesDir: File by lazy { extractDir(super.resourcesDir) }
-
-    override val includedDir: File by lazy { extractDir(super.includedDir) }
-
-    override val kotlinDir: File by lazy { extractDir(super.kotlinDir) }
-
-    override val nativeDir: File by lazy { extractDir(super.nativeDir) }
-
-    override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
-
-    override val irFile: File by lazy { extract(super.irFile) }
-
-    fun extract(file: File): File = zippedLibraryLayout.klibFile.withZipFileSystem { zipFileSystem ->
-        val temporary = createTempFile(file.name)
-        zipFileSystem.file(file).copyTo(temporary)
-        temporary.deleteOnExit()
-        temporary
+class BitcodeLibraryLayoutImpl(klib: File, target: KonanTarget?) : TargetedLibraryLayoutImpl(klib, target),
+    BitcodeKotlinLibraryLayout {
+    override val extractingToTemp: BitcodeKotlinLibraryLayout by lazy {
+        ExtractingBitcodeLibraryImpl(this)
     }
 
-    fun extractDir(directory: File): File = zippedLibraryLayout.klibFile.withZipFileSystem { zipFileSystem ->
-        val temporary = createTempDir(directory.name)
-        zipFileSystem.file(directory).recursiveCopyTo(temporary)
-        temporary.deleteOnExitRecursively()
-        temporary
-    }
+    override fun directlyFromZip(zipFileSystem: FileSystem): BitcodeKotlinLibraryLayout =
+        FromZipBitcodeLibraryImpl(this, zipFileSystem)
+
 }
 
-internal fun createKonanLibraryLayout(klib: File, target: KonanTarget? = null) =
-        if (klib.isFile) ZippedKonanLibraryLayout(klib, target) else UnzippedKonanLibraryLayout(klib, target)
+open class TargetedLibraryAccess<L : KotlinLibraryLayout>(klib: File, val target: KonanTarget?) :
+    BaseLibraryAccess<L>(klib) {
+    override val layout = TargetedLibraryLayoutImpl(klib, target)
+}
+
+open class BitcodeLibraryAccess<L : KotlinLibraryLayout>(klib: File, target: KonanTarget?) :
+    TargetedLibraryAccess<L>(klib, target) {
+    override val layout = BitcodeLibraryLayoutImpl(klib, target)
+}
+
+private open class FromZipTargetedLibraryImpl(zipped: TargetedLibraryLayoutImpl, zipFileSystem: FileSystem) :
+    FromZipBaseLibraryImpl(zipped, zipFileSystem), TargetedKotlinLibraryLayout
+
+private class FromZipBitcodeLibraryImpl(zipped: BitcodeLibraryLayoutImpl, zipFileSystem: FileSystem) :
+    FromZipTargetedLibraryImpl(zipped, zipFileSystem), BitcodeKotlinLibraryLayout
+
+open class ExtractingTargetedLibraryImpl(zipped: TargetedLibraryLayoutImpl) :
+    KotlinLibraryExtractor(zipped),
+    TargetedKotlinLibraryLayout by zipped {
+
+    override val includedDir: File by lazy { extractDir(zipped.includedDir) }
+}
+
+class ExtractingBitcodeLibraryImpl(zipped: BitcodeLibraryLayoutImpl) :
+    ExtractingTargetedLibraryImpl(zipped), BitcodeKotlinLibraryLayout {
+
+    override val kotlinDir: File by lazy { extractDir(zipped.kotlinDir) }
+    override val nativeDir: File by lazy { extractDir(zipped.nativeDir) }
+}

tools/kotlin-native-gradle-plugin/build.gradle

@@ -104,6 +104,7 @@ dependencies {
     bundleDependencies "org.jetbrains.kotlin:kotlin-gradle-plugin-api:$kotlinVersion"
     bundleDependencies "org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion"
     bundleDependencies "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
+    bundleDependencies "org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion"
 
     testImplementation 'junit:junit:4.12'
     testImplementation "org.jetbrains.kotlin:kotlin-test:$buildKotlinVersion"

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanLibrariesSpec.kt

@@ -24,7 +24,7 @@ import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.Internal
 import org.jetbrains.kotlin.gradle.plugin.tasks.KonanArtifactWithLibrariesTask
 import org.jetbrains.kotlin.gradle.plugin.tasks.KonanBuildingTask
-import org.jetbrains.kotlin.konan.CURRENT
+import org.jetbrains.kotlin.konan.*
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.library.SearchPathResolver
 import org.jetbrains.kotlin.konan.library.defaultResolver

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanPlugin.kt

@@ -91,7 +91,7 @@ internal val Project.konanHome: String
 
 internal val Project.konanVersion: KonanVersion
     get() = project.findProperty(KonanPlugin.ProjectProperty.KONAN_VERSION)
-        ?.toString()?.parseKonanVersion()
+        ?.toString()?.let { KonanVersion.fromString(it) }
         ?: KonanVersion.CURRENT
 
 internal val Project.konanBuildRoot          get() = buildDir.resolve("konan")

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/model/KonanModelImpl.kt

@@ -100,11 +100,11 @@ object KonanToolingModelBuilder : ToolingModelBuilder {
 }
 
 internal data class KonanModelImpl(
-        override val artifacts: List<KonanModelArtifact>,
-        override val konanHome: File,
-        override val konanVersion: KonanVersion,
-        override val languageVersion: String?,
-        override val apiVersion: String?
+    override val artifacts: List<KonanModelArtifact>,
+    override val konanHome: File,
+    override val konanVersion: KonanVersion,
+    override val languageVersion: String?,
+    override val apiVersion: String?
 ) : KonanModel
 
 internal data class KonanModelArtifactImpl(

utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/InteropCompiler.kt

@@ -9,10 +9,10 @@ import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.konan.library.*
 import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.native.interop.gen.jvm.interop
-import org.jetbrains.kotlin.utils.addIfNotNull
 import org.jetbrains.kliopt.ArgParser
 import org.jetbrains.kotlin.konan.CURRENT
 import org.jetbrains.kotlin.konan.KonanVersion
+import org.jetbrains.kotlin.library.toUnresolvedLibraries
 import org.jetbrains.kotlin.native.interop.tool.*
 
 // TODO: this function should eventually be eliminated from 'utilities'.