From ccf084b1a55cf383feb6dbba12c1c982cfac5762 Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Tue, 14 Jan 2020 15:58:48 +0300 Subject: [PATCH] Initial implementation of klib components --- .../cli/metadata/K2MetadataKlibSerializer.kt | 6 ++- .../kotlin/ir/backend/js/JsLibraryResolver.kt | 3 +- .../jetbrains/kotlin/ir/backend/js/klib.kt | 6 ++- .../jetbrains/kotlin/library/KotlinLibrary.kt | 2 + .../kotlin/library/KotlinLibraryLayout.kt | 13 ++++-- .../kotlin/library/SearchPathResolver.kt | 23 ++++++++--- .../kotlin/library/impl/KotlinLibraryImpl.kt | 40 +++++++++++++++++-- .../library/impl/KotlinLibraryLayoutImpl.kt | 20 +++++----- .../library/impl/KotlinLibraryWriterImpl.kt | 5 ++- .../kotlin/konan/target/Distribution.kt | 3 +- 10 files changed, 92 insertions(+), 29 deletions(-) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt index 66f365ccb1f..ac26232f61d 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt @@ -30,7 +30,7 @@ import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.KotlinLibraryVersioning import org.jetbrains.kotlin.library.SerializedMetadata import org.jetbrains.kotlin.library.impl.buildKoltinLibrary -import org.jetbrains.kotlin.library.impl.createKotlinLibrary +import org.jetbrains.kotlin.library.impl.createKotlinLibraryComponents import org.jetbrains.kotlin.library.metadata.parseModuleHeader import org.jetbrains.kotlin.metadata.builtins.BuiltInsBinaryVersion import org.jetbrains.kotlin.name.Name @@ -110,7 +110,9 @@ private class KlibMetadataDependencyContainer( val klibFiles = classpathFiles .filter { it.extension == "klib" || it.isDirectory } - klibFiles.map { createKotlinLibrary(org.jetbrains.kotlin.konan.file.File(it.absolutePath)) } + // TODO: need to move K2Metadata to SearchPathResolver. + // Otherewise there's no good way to choose between the components. + klibFiles.map { createKotlinLibraryComponents(org.jetbrains.kotlin.konan.file.File(it.absolutePath)).single() } } private val builtIns diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsLibraryResolver.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsLibraryResolver.kt index e9997a6670c..d28b796f928 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsLibraryResolver.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsLibraryResolver.kt @@ -12,6 +12,7 @@ import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.KotlinLibraryProperResolverWithAttributes import org.jetbrains.kotlin.library.UnresolvedLibrary import org.jetbrains.kotlin.library.impl.createKotlinLibrary +import org.jetbrains.kotlin.library.impl.createKotlinLibraryComponents import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult import org.jetbrains.kotlin.library.resolver.impl.libraryResolver import org.jetbrains.kotlin.util.Logger @@ -37,7 +38,7 @@ class JsLibraryResolver( emptyList() ) { // Stick with the default KotlinLibrary for now. - override fun libraryBuilder(file: File, isDefault: Boolean) = createKotlinLibrary(file, isDefault) + override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKotlinLibraryComponents(file, isDefault) } // TODO: This is a temporary set of library resolver policies for js compiler. diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index 32fccfe1b9c..21238c3f74b 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -43,6 +43,7 @@ import org.jetbrains.kotlin.konan.util.KlibMetadataFactories import org.jetbrains.kotlin.library.* import org.jetbrains.kotlin.library.impl.buildKoltinLibrary import org.jetbrains.kotlin.library.impl.createKotlinLibrary +import org.jetbrains.kotlin.library.impl.createKotlinLibraryComponents import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.metadata.ProtoBuf @@ -73,8 +74,11 @@ val KotlinLibrary.isBuiltIns: Boolean .propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true) .isEmpty() +// TODO: The only place loadKlib() is used now is Wasm backend. +// Need to move it to SearchPathResolver too. +// Otherwise there's no good way to choose between the components. fun loadKlib(klibPath: String) = - createKotlinLibrary(KFile(KFile(klibPath).absolutePath)) + createKotlinLibraryComponents(KFile(KFile(klibPath).absolutePath)).single() val emptyLoggingContext = object : LoggingContext { override var inVerbosePhase = false diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt index 490e7c3c193..7e9ff568c2e 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt @@ -23,10 +23,12 @@ const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations interface BaseKotlinLibrary { val libraryName: String val libraryFile: File + val componentList: List val versions: KotlinLibraryVersioning // Whether this library is default (provided by distribution)? val isDefault: Boolean val manifestProperties: Properties + val has_pre_1_4_manifest: Boolean } interface MetadataLibrary { diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt index 2469418379c..aa243a584c1 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt @@ -30,15 +30,20 @@ interface KotlinLibraryLayout { val libDir: File val libraryName: String get() = libDir.path + val component: String? + val componentDir: File + get() = File(libDir, component!!) val manifestFile - get() = File(libDir, KLIB_MANIFEST_FILE_NAME) + get() = File(componentDir, KLIB_MANIFEST_FILE_NAME) val resourcesDir - get() = File(libDir, "resources") + get() = File(componentDir, "resources") + val pre_1_4_manifest: File + get() = File(libDir, KLIB_MANIFEST_FILE_NAME) } interface MetadataKotlinLibraryLayout : KotlinLibraryLayout { val metadataDir - get() = File(libDir, "linkdata") + get() = File(componentDir, "linkdata") val moduleHeaderFile get() = File(metadataDir, KLIB_MODULE_METADATA_FILE_NAME) @@ -51,7 +56,7 @@ interface MetadataKotlinLibraryLayout : KotlinLibraryLayout { interface IrKotlinLibraryLayout : KotlinLibraryLayout { val irDir - get() = File(libDir, KLIB_IR_FOLDER_NAME) + get() = File(componentDir, KLIB_IR_FOLDER_NAME) val irDeclarations get() = File(irDir, "irDeclarations.knd") val irSymbols diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/SearchPathResolver.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/SearchPathResolver.kt index 6c3fda812be..f4106e4acf7 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/SearchPathResolver.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/SearchPathResolver.kt @@ -2,6 +2,7 @@ package org.jetbrains.kotlin.library import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.library.impl.isPre_1_4_Library import org.jetbrains.kotlin.util.* const val KOTLIN_STDLIB_NAME = "stdlib" @@ -21,7 +22,7 @@ interface SearchPathResolverWithAttributes: SearchPathResolver } // This is a simple library resolver that only cares for file names. -abstract class KotlinLibrarySearchPathResolver( +abstract class KotlinLibrarySearchPathResolver ( repositories: List, directLibs: List, val distributionKlib: String?, @@ -43,10 +44,10 @@ abstract class KotlinLibrarySearchPathResolver( private val repoRoots: List by lazy { repositories.map { File(it) } } - abstract fun libraryBuilder(file: File, isDefault: Boolean): L + abstract fun libraryComponentBuilder(file: File, isDefault: Boolean): List private val directLibraries: List by lazy { - directLibs.mapNotNull { found(File(it)) }.map { libraryBuilder(it, false) } + directLibs.mapNotNull { found(File(it)) }.flatMap { libraryComponentBuilder(it, false) } } // This is the place where we specify the order of library search. @@ -55,8 +56,7 @@ abstract class KotlinLibrarySearchPathResolver( } private fun found(candidate: File): File? { - fun check(file: File): Boolean = - file.exists && (file.isFile || File(file, "manifest").exists) + fun check(file: File): Boolean = file.exists val noSuffix = File(candidate.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)) val withSuffix = File(candidate.path.suffixIfNot(KLIB_FILE_EXTENSION_WITH_DOT)) @@ -94,11 +94,22 @@ abstract class KotlinLibrarySearchPathResolver( return sequence.filterNotNull() } + private fun Sequence.filterOutPre_1_4_libraries(): Sequence = this.filter{ + if (it.isPre_1_4_Library) { + logger.warning("Skipping \"$it\" as it is a pre 1.4 library") + false + } else { + true + } + } + override fun resolve(unresolved: UnresolvedLibrary, isDefaultLink: Boolean): L { val givenPath = unresolved.path try { val fileSequence = resolutionSequence(givenPath) - val matching = fileSequence.map { libraryBuilder(it, isDefaultLink) } + val matching = fileSequence + .filterOutPre_1_4_libraries() + .flatMap { libraryComponentBuilder(it, isDefaultLink).asSequence() } .map { it.takeIf { libraryMatch(it, unresolved) } } .filterNotNull() diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt index 848b4f9900f..ee98b939c15 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt @@ -28,8 +28,21 @@ open class BaseKotlinLibraryImpl( override val libraryFile get() = access.klib override val libraryName: String by lazy { access.inPlace { it.libraryName } } + override val componentList: List by lazy { + access.inPlace { + it.libDir.listFiles + .filter { it.isDirectory } + .filter { it.listFiles.map { it.name }.contains(KLIB_MANIFEST_FILE_NAME) } + .map { it.name } + } + } + override fun toString() = "$libraryName[default=$isDefault]" + override val has_pre_1_4_manifest: Boolean by lazy { + access.inPlace { it.pre_1_4_manifest.exists } + } + override val manifestProperties: Properties by lazy { access.inPlace { it.manifestFile.loadProperties() } } @@ -221,11 +234,12 @@ open class KotlinLibraryImpl( fun createKotlinLibrary( libraryFile: File, + component: String, isDefault: Boolean = false ): KotlinLibrary { - val baseAccess = BaseLibraryAccess(libraryFile) - val metadataAccess = MetadataLibraryAccess(libraryFile) - val irAccess = IrLibraryAccess(libraryFile) + val baseAccess = BaseLibraryAccess(libraryFile, component) + val metadataAccess = MetadataLibraryAccess(libraryFile, component) + val irAccess = IrLibraryAccess(libraryFile, component) val base = BaseKotlinLibraryImpl(baseAccess, isDefault) val metadata = MetadataLibraryImpl(metadataAccess) @@ -233,4 +247,22 @@ fun createKotlinLibrary( // val ir = IrPerFileLibraryImpl(irAccess) return KotlinLibraryImpl(base, metadata, ir) -} \ No newline at end of file +} + +fun createKotlinLibraryComponents( + libraryFile: File, + isDefault: Boolean = true +) : List { + val baseAccess = BaseLibraryAccess(libraryFile, null) + val base = BaseKotlinLibraryImpl(baseAccess, isDefault) + return base.componentList.map { + createKotlinLibrary(libraryFile, it, isDefault) + } +} + +val File.isPre_1_4_Library: Boolean + get() { + val baseAccess = BaseLibraryAccess(this, null) + val base = BaseKotlinLibraryImpl(baseAccess, false) + return base.has_pre_1_4_manifest + } \ No newline at end of file diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryLayoutImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryLayoutImpl.kt index dd8b7eaa97f..49893b8e66a 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryLayoutImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryLayoutImpl.kt @@ -7,7 +7,7 @@ import org.jetbrains.kotlin.library.* import org.jetbrains.kotlin.util.removeSuffixIfPresent import java.nio.file.FileSystem -open class KotlinLibraryLayoutImpl(val klib: File) : KotlinLibraryLayout { +open class KotlinLibraryLayoutImpl(val klib: File, override val component: String?) : KotlinLibraryLayout { val isZipped = klib.isFile init { @@ -32,7 +32,7 @@ open class KotlinLibraryLayoutImpl(val klib: File) : KotlinLibraryLayout { } -class MetadataLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), MetadataKotlinLibraryLayout { +class MetadataLibraryLayoutImpl(klib: File, component: String) : KotlinLibraryLayoutImpl(klib, component), MetadataKotlinLibraryLayout { override val extractingToTemp: MetadataKotlinLibraryLayout by lazy { ExtractingMetadataLibraryImpl(this) @@ -42,7 +42,7 @@ class MetadataLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), Met FromZipMetadataLibraryImpl(this, zipFileSystem) } -class IrLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), IrKotlinLibraryLayout { +class IrLibraryLayoutImpl(klib: File, component: String) : KotlinLibraryLayoutImpl(klib, component), IrKotlinLibraryLayout { override val extractingToTemp: IrKotlinLibraryLayout by lazy { ExtractingIrLibraryImpl(this) @@ -52,8 +52,8 @@ class IrLibraryLayoutImpl(klib: File) : KotlinLibraryLayoutImpl(klib), IrKotlinL FromZipIrLibraryImpl(this, zipFileSystem) } -open class BaseLibraryAccess(val klib: File) { - open val layout = KotlinLibraryLayoutImpl(klib) +open class BaseLibraryAccess(val klib: File, component: String?) { + open val layout = KotlinLibraryLayoutImpl(klib, component) fun realFiles(action: (L) -> T): T = if (layout.isZipped) @@ -71,12 +71,12 @@ open class BaseLibraryAccess(val klib: File) { } -open class MetadataLibraryAccess(klib: File) : BaseLibraryAccess(klib) { - override val layout = MetadataLibraryLayoutImpl(klib) +open class MetadataLibraryAccess(klib: File, component: String) : BaseLibraryAccess(klib, component) { + override val layout = MetadataLibraryLayoutImpl(klib, component) } -open class IrLibraryAccess(klib: File) : BaseLibraryAccess(klib) { - override val layout = IrLibraryLayoutImpl(klib) +open class IrLibraryAccess(klib: File, component: String) : BaseLibraryAccess(klib, component) { + override val layout = IrLibraryLayoutImpl(klib, component) } open class FromZipBaseLibraryImpl(zipped: KotlinLibraryLayoutImpl, zipFileSystem: FileSystem) : @@ -84,6 +84,7 @@ open class FromZipBaseLibraryImpl(zipped: KotlinLibraryLayoutImpl, zipFileSystem override val libraryName = zipped.libraryName override val libDir = zipFileSystem.file(zipped.libDir) + override val component = zipped.component } class FromZipMetadataLibraryImpl(zipped: MetadataLibraryLayoutImpl, zipFileSystem: FileSystem) : @@ -113,6 +114,7 @@ fun KotlinLibraryLayoutImpl.extractDir(directory: File): File = this.klib.withZi open class ExtractingKotlinLibraryLayout(zipped: KotlinLibraryLayoutImpl) : KotlinLibraryLayout { override val libDir: File get() = error("Extracting layout doesn't extract its own root") override val libraryName = zipped.libraryName + override val component = zipped.component } open class ExtractingBaseLibraryImpl(zipped: KotlinLibraryLayoutImpl) : diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt index 2fadef786e3..88736ea430d 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt @@ -11,8 +11,11 @@ import org.jetbrains.kotlin.konan.properties.Properties import org.jetbrains.kotlin.konan.properties.saveToFile import org.jetbrains.kotlin.library.* +const val KLIB_DEFAULT_COMPONENT_NAME = "default" + open class KotlinLibraryLayoutForWriter( - override val libDir: File + override val libDir: File, + override val component: String = KLIB_DEFAULT_COMPONENT_NAME ) : KotlinLibraryLayout, MetadataKotlinLibraryLayout, IrKotlinLibraryLayout open class BaseWriterImpl( diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt b/konan/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt index 9dae3f5a2d7..cd4712462c3 100644 --- a/konan/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt +++ b/konan/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt @@ -68,10 +68,11 @@ class Distribution( val klib = "$konanHome/klib" val stdlib = "$klib/common/stdlib" + val stdlibDefaultComponent = "$stdlib/default" fun defaultNatives(target: KonanTarget) = "$konanHome/konan/targets/${target.visibleName}/native" - fun runtime(target: KonanTarget) = runtimeFileOverride ?: "$stdlib/targets/${target.visibleName}/native/runtime.bc" + fun runtime(target: KonanTarget) = runtimeFileOverride ?: "$stdlibDefaultComponent/targets/${target.visibleName}/native/runtime.bc" val launcherFiles = listOf("launcher.bc")