diff --git a/build.gradle.kts b/build.gradle.kts index 53c91650853..86d145a1f06 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -768,6 +768,7 @@ tasks { register("toolsTest") { dependsOn(":tools:kotlinp:test") dependsOn(":native:kotlin-klib-commonizer:test") + dependsOn(":native:kotlin-klib-commonizer-api:test") } register("examplesTest") { diff --git a/buildSrc/src/main/kotlin/tasks.kt b/buildSrc/src/main/kotlin/tasks.kt index 58b37c62099..51a47280ad4 100644 --- a/buildSrc/src/main/kotlin/tasks.kt +++ b/buildSrc/src/main/kotlin/tasks.kt @@ -53,7 +53,8 @@ fun Task.dependsOnKotlinPluginInstall() { ":kotlin-scripting-compiler-embeddable:install", ":kotlin-scripting-compiler-impl-embeddable:install", ":kotlin-test-js-runner:install", - ":native:kotlin-klib-commonizer-embeddable:install" + ":native:kotlin-klib-commonizer-embeddable:install", + ":native:kotlin-klib-commonizer-api:install" ) } diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index d0f46edc9ee..74be09e504c 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -42,6 +42,7 @@ dependencies { implementation(kotlinStdlib()) implementation(project(":kotlin-util-klib")) + implementation(project(":native:kotlin-klib-commonizer-api")) compileOnly(project(":kotlin-reflect-api")) compileOnly(project(":kotlin-android-extensions")) compileOnly(project(":kotlin-build-common")) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCliCommonizer.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCliCommonizer.kt new file mode 100644 index 00000000000..0d6f6c9335d --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCliCommonizer.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("FunctionName") + +package org.jetbrains.kotlin.compilerRunner + +import org.gradle.api.Project +import org.jetbrains.kotlin.descriptors.commonizer.CliCommonizer + +/** + * Creates an instance of [CliCommonizer] that is backed by [KotlinNativeCommonizerToolRunner] to adhere to user defined settings + * when executing the commonizer (like jvm arguments, running in separate process, etc) + */ +internal fun GradleCliCommonizer(project: Project): CliCommonizer { + return CliCommonizer(CliCommonizer.Executor { arguments -> + KotlinNativeCommonizerToolRunner(project).run(arguments) + }) +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeKlibCommonizerToolRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt similarity index 93% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeKlibCommonizerToolRunner.kt rename to libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt index c8a0bfa7309..dff26f69c67 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeKlibCommonizerToolRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt @@ -11,7 +11,7 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion import java.io.File -internal class KotlinNativeKlibCommonizerToolRunner(project: Project) : KotlinToolRunner(project) { +internal class KotlinNativeCommonizerToolRunner(project: Project) : KotlinToolRunner(project) { override val displayName get() = "Kotlin/Native KLIB commonizer" override val mainClass: String get() = "org.jetbrains.kotlin.descriptors.commonizer.cli.CommonizerCLI" diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt index 2da6a510bdb..984ec24de82 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt @@ -8,7 +8,7 @@ package org.jetbrains.kotlin.gradle.targets.native.internal import org.gradle.api.DefaultTask import org.gradle.api.Project import org.gradle.api.tasks.* -import org.jetbrains.kotlin.compilerRunner.KotlinNativeKlibCommonizerToolRunner +import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion import org.jetbrains.kotlin.gradle.targets.native.internal.SuccessMarker.Companion.getSuccessMarker @@ -24,7 +24,6 @@ import java.nio.file.* import java.nio.file.attribute.* import java.time.* import java.util.* -import javax.inject.Inject internal const val COMMONIZER_TASK_NAME = "runCommonizer" @@ -190,7 +189,7 @@ internal fun Project.createTempNativeDistributionCommonizerOutputDirectory(targe fun callCommonizerCLI(project: Project, commandLineArguments: List) { if (commandLineArguments.isEmpty()) return - KotlinNativeKlibCommonizerToolRunner(project).run(commandLineArguments) + KotlinNativeCommonizerToolRunner(project).run(commandLineArguments) } private fun renameDirectory(source: File, destination: File) { diff --git a/native/commonizer-api/build.gradle.kts b/native/commonizer-api/build.gradle.kts new file mode 100644 index 00000000000..314a2b16402 --- /dev/null +++ b/native/commonizer-api/build.gradle.kts @@ -0,0 +1,44 @@ +import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader + +plugins { + kotlin("jvm") + id("jps-compatible") +} + +kotlin { + explicitApi() +} + +description = "Kotlin KLIB Library Commonizer API" +publish() + +dependencies { + implementation(kotlinStdlib()) + implementation(project(":native:kotlin-native-utils")) + testImplementation(project(":kotlin-test::kotlin-test-junit")) + testImplementation(commonDep("junit:junit")) + testImplementation(projectTests(":compiler:tests-common")) + testRuntimeOnly(project(":native:kotlin-klib-commonizer")) +} + +sourceSets { + "main" { projectDefault() } + "test" { projectDefault() } +} + +tasks.register("downloadNativeCompiler") { + doFirst { + NativeCompilerDownloader(project).downloadIfNeeded() + } +} + +projectTest(parallel = false) { + dependsOn(":dist") + dependsOn("downloadNativeCompiler") + workingDir = projectDir + environment("KONAN_HOME", NativeCompilerDownloader(project).compilerDirectory.absolutePath) +} + +runtimeJar() +sourcesJar() +javadocJar() diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizer.kt b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizer.kt new file mode 100644 index 00000000000..bee9c28084f --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizer.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import java.io.File +import java.net.URLClassLoader +import kotlin.jvm.Throws + +public fun CliCommonizer(classpath: Iterable): CliCommonizer { + return CliCommonizer(URLClassLoader(classpath.map { it.absoluteFile.toURI().toURL() }.toTypedArray())) +} + +public fun CliCommonizer(classLoader: ClassLoader): CliCommonizer { + return CliCommonizer(CommonizerClassLoaderExecutor(classLoader)) +} + +public class CliCommonizer(private val executor: Executor) : Commonizer { + + public fun interface Executor { + public operator fun invoke(arguments: List) + } + + override fun commonizeLibraries( + konanHome: File, + inputLibraries: Set, + dependencyLibraries: Set, + outputCommonizerTarget: SharedCommonizerTarget, + outputDirectory: File + ) { + val arguments = mutableListOf().apply { + add("native-klib-commonize") + add("-distribution-path"); add(konanHome.absolutePath) + add("-input-libraries"); add(inputLibraries.joinToString(";") { it.absolutePath }) + add("-dependency-libraries"); add(dependencyLibraries.joinToString(";") { it.absolutePath }) + add("-output-commonizer-target"); add(outputCommonizerTarget.identityString) + add("-output-path"); add(outputDirectory.absolutePath) + } + executor(arguments) + } +} + +private class CommonizerClassLoaderExecutor(private val commonizerClassLoader: ClassLoader) : CliCommonizer.Executor { + companion object { + private const val commonizerMainClass = "org.jetbrains.kotlin.descriptors.commonizer.cli.CommonizerCLI" + private const val commonizerMainFunction = "main" + } + + @Throws(Throwable::class) + override fun invoke(arguments: List) { + val commonizerMainClass = commonizerClassLoader.loadClass(commonizerMainClass) + val commonizerMainMethod = commonizerMainClass.methods.singleOrNull { it.name == commonizerMainFunction } + ?: throw IllegalArgumentException( + "Missing or conflicting $commonizerMainFunction function in " + + "Class ${commonizerMainClass.name} from ClassLoader $commonizerClassLoader" + ) + commonizerMainMethod.invoke(null, arguments.toTypedArray()) + } +} diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/Commonizer.kt b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/Commonizer.kt new file mode 100644 index 00000000000..6795dad584f --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/Commonizer.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import java.io.File +import java.io.Serializable +import kotlin.jvm.Throws + +public interface Commonizer : Serializable { + @Throws(Throwable::class) + public fun commonizeLibraries( + konanHome: File, + inputLibraries: Set, + dependencyLibraries: Set, + outputCommonizerTarget: SharedCommonizerTarget, + outputDirectory: File + ) +} diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt new file mode 100644 index 00000000000..35328d0f057 --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("FunctionName") + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.konan.target.KonanTarget +import java.io.Serializable + +// N.B. TargetPlatform/SimplePlatform are non exhaustive enough to address both target platforms such as +// JVM, JS and concrete Kotlin/Native targets, e.g. macos_x64, ios_x64, linux_x64. +public sealed class CommonizerTarget : Serializable { + final override fun toString(): String = identityString +} + +public data class LeafCommonizerTarget public constructor(val name: String) : CommonizerTarget() { + public constructor(konanTarget: KonanTarget) : this(konanTarget.name) + + public val konanTargetOrNull: KonanTarget? = KonanTarget.predefinedTargets[name] + + public val konanTarget: KonanTarget get() = konanTargetOrNull ?: error("Unknown KonanTarget: $name") +} + +public data class SharedCommonizerTarget(val targets: Set) : CommonizerTarget() { + public constructor(vararg targets: CommonizerTarget) : this(targets.toSet()) + public constructor(vararg targets: KonanTarget) : this(targets.toSet()) + public constructor(targets: Iterable) : this(targets.map(::LeafCommonizerTarget).toSet()) + + init { + require(targets.isNotEmpty()) + } +} + +public fun CommonizerTarget(konanTargets: Iterable): CommonizerTarget { + val konanTargetsSet = konanTargets.toSet() + require(konanTargetsSet.isNotEmpty()) { "Empty set of of konanTargets" } + val leafTargets = konanTargetsSet.map(::LeafCommonizerTarget) + return leafTargets.singleOrNull() ?: SharedCommonizerTarget(leafTargets.toSet()) +} + +public fun CommonizerTarget(konanTarget: KonanTarget): LeafCommonizerTarget { + return LeafCommonizerTarget(konanTarget) +} + +public fun CommonizerTarget(konanTarget: KonanTarget, vararg konanTargets: KonanTarget): SharedCommonizerTarget { + val targets = ArrayList(konanTargets.size + 1).apply { + add(konanTarget) + addAll(konanTargets) + } + return SharedCommonizerTarget(targets.map(::LeafCommonizerTarget).toSet()) +} + +public val CommonizerTarget.identityString: String + get() = when (this) { + is LeafCommonizerTarget -> name + is SharedCommonizerTarget -> identityString + } + +private val SharedCommonizerTarget.identityString: String + get() { + val segments = targets.map(CommonizerTarget::identityString).sorted() + return segments.joinToString( + separator = ", ", prefix = "(", postfix = ")" + ) + } + +public val CommonizerTarget.prettyName: String + get() = when (this) { + is LeafCommonizerTarget -> "[$name]" + is SharedCommonizerTarget -> prettyName(null) + } + +public fun SharedCommonizerTarget.prettyName(highlightedChild: CommonizerTarget?): String { + return targets + .sortedWith(compareBy { it.level }.thenBy { it.identityString }).joinToString(", ", "[", "]") { child -> + when (child) { + is LeafCommonizerTarget -> child.name + is SharedCommonizerTarget -> child.prettyName(highlightedChild) + } + if (child == highlightedChild) "(*)" else "" + } +} + +public val CommonizerTarget.konanTargets: Set + get() { + return when (this) { + is LeafCommonizerTarget -> setOf(konanTarget) + is SharedCommonizerTarget -> targets.flatMap { it.konanTargets }.toSet() + } + } + +public val CommonizerTarget.level: Int + get() { + return when (this) { + is LeafCommonizerTarget -> return 0 + is SharedCommonizerTarget -> targets.maxOf { it.level } + 1 + } + } diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/TargetLibrariesLayout.kt b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/TargetLibrariesLayout.kt new file mode 100644 index 00000000000..63a862f9b8c --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/TargetLibrariesLayout.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR +import java.io.File + + +public fun interface CommonizerOutputLayout { + public fun getTargetDirectory(root: File, target: CommonizerTarget): File +} + +public object NativeDistributionCommonizerOutputLayout : CommonizerOutputLayout { + override fun getTargetDirectory(root: File, target: CommonizerTarget): File { + return when (target) { + is LeafCommonizerTarget -> root.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(target.name) + is SharedCommonizerTarget -> root.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) + } + } +} + +public object HierarchicalCommonizerOutputLayout : CommonizerOutputLayout { + override fun getTargetDirectory(root: File, target: CommonizerTarget): File { + return root.resolve(target.identityString) + } +} diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/parseCommonizerTarget.kt b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/parseCommonizerTarget.kt new file mode 100644 index 00000000000..5c4e570b929 --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/descriptors/commonizer/parseCommonizerTarget.kt @@ -0,0 +1,196 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.commonizer.IdentityStringSyntaxNode.LeafTargetSyntaxNode +import org.jetbrains.kotlin.descriptors.commonizer.IdentityStringSyntaxNode.SharedTargetSyntaxNode +import org.jetbrains.kotlin.descriptors.commonizer.IdentityStringToken.* + +public fun parseCommonizerTarget(identityString: String): CommonizerTarget { + try { + val tokens = tokenizeIdentityString(identityString) + val syntaxTree = parser(tokens) ?: error("Failed building syntax tree. $identityString") + check(syntaxTree.remaining.isEmpty()) { "Failed building syntax tree. Unexpected remaining tokens ${syntaxTree.remaining}" } + return buildCommonizerTarget(syntaxTree.value) + } catch (e: Throwable) { + throw IllegalArgumentException("Failed parsing CommonizerTarget from \"$identityString\"", e) + } +} + +//region Tokens + +private fun tokenizeIdentityString(identityString: String): List { + var remainingString = identityString + val tokenizer = sharedTargetStartTokenizer + sharedTargetEndTokenizer + separatorTokenizer + wordTokenizer + return mutableListOf().apply { + while (remainingString.isNotEmpty()) { + val generatedToken = tokenizer.nextToken(remainingString) + ?: error("Unexpected token at $remainingString") + + remainingString = generatedToken.remaining + add(generatedToken.token) + } + }.toList() +} + +private sealed class IdentityStringToken { + data class Word(val value: String) : IdentityStringToken() + object Separator : IdentityStringToken() + object SharedTargetStart : IdentityStringToken() + object SharedTargetEnd : IdentityStringToken() + + final override fun toString(): String { + return when (this) { + is Word -> value + is Separator -> ", " + is SharedTargetStart -> "[" + is SharedTargetEnd -> "]" + } + } +} + +private data class GeneratedToken(val token: IdentityStringToken, val remaining: String) + +private interface IdentityStringTokenizer { + fun nextToken(value: String): GeneratedToken? +} + +private operator fun IdentityStringTokenizer.plus(other: IdentityStringTokenizer): IdentityStringTokenizer { + return CompositeIdentityStringTokenizer(this, other) +} + +private data class CompositeIdentityStringTokenizer( + val first: IdentityStringTokenizer, + val second: IdentityStringTokenizer +) : IdentityStringTokenizer { + override fun nextToken(value: String): GeneratedToken? { + return first.nextToken(value) ?: second.nextToken(value) + } +} + +private data class RegexIdentityStringTokenizer( + val regex: Regex, + val token: (String) -> IdentityStringToken +) : IdentityStringTokenizer { + override fun nextToken(value: String): GeneratedToken? { + val firstMatchResult = regex.findAll(value, 0).firstOrNull() ?: return null + val range = firstMatchResult.range + if (range.first != 0) return null + return GeneratedToken(token(firstMatchResult.value), value.drop(firstMatchResult.value.length)) + } +} + +private val sharedTargetStartTokenizer = + RegexIdentityStringTokenizer(Regex.fromLiteral("(")) { SharedTargetStart } + +private val sharedTargetEndTokenizer = + RegexIdentityStringTokenizer(Regex.fromLiteral(")")) { SharedTargetEnd } + +private val separatorTokenizer = + RegexIdentityStringTokenizer(Regex("""\s*,\s*""")) { Separator } + +private val wordTokenizer = + RegexIdentityStringTokenizer(Regex("\\w+"), IdentityStringToken::Word) + +//endregion + +//region Syntax Tree + +private val parser = anyOf(SharedTargetParser, LeafTargetParser) + +private data class ParserOutput(val value: T, val remaining: List) + +private interface Parser { + operator fun invoke(tokens: List): ParserOutput? +} + + +private fun anyOf(vararg parser: Parser): Parser { + return AnyOfParser(parser.toList()) +} + +private data class AnyOfParser(val parsers: List>) : Parser { + override fun invoke(tokens: List): ParserOutput? { + return parsers.mapNotNull { parser -> parser(tokens) }.firstOrNull() + } +} + +private fun Parser.oneOrMore(): Parser> { + return OneOrMoreParser(this) +} + +private data class OneOrMoreParser(val parser: Parser) : Parser> { + override fun invoke(tokens: List): ParserOutput>? { + val outputs = mutableListOf() + var remainingTokens = tokens + while (true) { + val output = parser(remainingTokens) ?: break + if (output.remaining == remainingTokens) break + outputs.add(output.value) + remainingTokens = output.remaining + } + if (outputs.isEmpty()) { + return null + } + return ParserOutput(outputs.toList(), remainingTokens) + } +} + +private fun Parser.ignore(token: IdentityStringToken): Parser { + return IgnoreTokensParser(this, token) +} + +private data class IgnoreTokensParser(val parser: Parser, val ignoredToken: IdentityStringToken) : Parser { + override fun invoke(tokens: List): ParserOutput? { + return parser( + if (tokens.firstOrNull() == ignoredToken) tokens.drop(1) else tokens + ) + } +} + +private object LeafTargetParser : Parser { + override fun invoke(tokens: List): ParserOutput? { + val nextToken = tokens.firstOrNull() as? Word ?: return null + return ParserOutput(LeafTargetSyntaxNode(nextToken), tokens.drop(1)) + } +} + +private object SharedTargetParser : Parser { + override fun invoke(tokens: List): ParserOutput? { + if (tokens.firstOrNull() !is SharedTargetStart) return null + + val innerParser = anyOf(LeafTargetParser, SharedTargetParser).ignore(Separator).oneOrMore() + val innerParserOutput = innerParser(tokens.drop(1)) ?: return null + + val closingToken = innerParserOutput.remaining.firstOrNull() + if (closingToken != SharedTargetEnd) { + error("Missing ']' at ${tokens.joinToString("")}") + } + + return ParserOutput(SharedTargetSyntaxNode(innerParserOutput.value), innerParserOutput.remaining.drop(1)) + } + +} + +private sealed class IdentityStringSyntaxNode { + data class LeafTargetSyntaxNode(val token: Word) : IdentityStringSyntaxNode() + data class SharedTargetSyntaxNode(val children: List) : IdentityStringSyntaxNode() +} + +//endregion Tree + +//region Build CommonizerTarget + +private fun buildCommonizerTarget(node: IdentityStringSyntaxNode): CommonizerTarget { + return when (node) { + is LeafTargetSyntaxNode -> LeafCommonizerTarget(node.token.value) + is SharedTargetSyntaxNode -> SharedCommonizerTarget( + node.children.map { child -> buildCommonizerTarget(child) }.toSet() + ) + } +} + +//endregion diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizerTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizerTest.kt new file mode 100644 index 00000000000..e51313d4a1b --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CliCommonizerTest.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.commonizer.utils.konanHome +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import kotlin.test.Test + +class CliCommonizerTest { + + @get:Rule + val temporaryOutputDirectory = TemporaryFolder() + + @Test + fun invokeCliWithEmptyArguments() { + val commonizer = CliCommonizer(this::class.java.classLoader) + commonizer.commonizeLibraries( + konanHome = konanHome, + inputLibraries = emptySet(), + dependencyLibraries = emptySet(), + outputCommonizerTarget = CommonizerTarget(KonanTarget.LINUX_X64, KonanTarget.MACOS_X64), + outputDirectory = temporaryOutputDirectory.root + ) + } +} diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizeLibcurlTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizeLibcurlTest.kt new file mode 100644 index 00000000000..8a3191e5bf9 --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizeLibcurlTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.commonizer.utils.konanHome +import org.jetbrains.kotlin.konan.target.KonanTarget.LINUX_ARM64 +import org.jetbrains.kotlin.konan.target.KonanTarget.LINUX_X64 +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertTrue +import kotlin.test.fail + +class CommonizeLibcurlTest { + + @get:Rule + val temporaryOutputDirectory = TemporaryFolder() + + @Test + fun commonizeSuccessfully() { + val libraries = File("testData/libcurl").walkTopDown().filter { it.isFile && it.extension == "klib" }.toSet() + val commonizer = CliCommonizer(this::class.java.classLoader) + + commonizer.commonizeLibraries( + konanHome = konanHome, + inputLibraries = libraries, + dependencyLibraries = emptySet(), + outputCommonizerTarget = CommonizerTarget(LINUX_ARM64, LINUX_X64), + outputDirectory = temporaryOutputDirectory.root + ) + + val x64OutputDirectory = temporaryOutputDirectory.root.resolve(CommonizerTarget(LINUX_X64).identityString) + val arm64OutputDirectory = temporaryOutputDirectory.root.resolve(CommonizerTarget(LINUX_ARM64).identityString) + val commonOutputDirectory = temporaryOutputDirectory.root.resolve(CommonizerTarget(LINUX_X64, LINUX_ARM64).identityString) + + assertTrue( + x64OutputDirectory.exists(), + "Missing output directory for x64 target" + ) + + assertTrue( + arm64OutputDirectory.exists(), + "Missing output directory for arm64 target" + ) + + assertTrue( + commonOutputDirectory.exists(), + "Missing output directory for commonized x64&arm64 target" + ) + + fun assertContainsKnmFiles(file: File) { + assertTrue( + file.walkTopDown().any { it.extension == "knm" }, + "Expected directory ${file.name} to contain at least one knm file" + ) + } + + assertContainsKnmFiles(x64OutputDirectory) + assertContainsKnmFiles(arm64OutputDirectory) + assertContainsKnmFiles(commonOutputDirectory) + + fun assertContainsManifestWithContent(directory: File, content: String) { + val manifest = directory.walkTopDown().firstOrNull { it.name == "manifest" } + ?: fail("${directory.name} does not contain any manifest") + + assertTrue( + content in manifest.readText(), + "Expected manifest in ${directory.name} to contain $content\n${manifest.readText()}" + ) + } + + assertContainsManifestWithContent(x64OutputDirectory, "native_targets=linux_x64") + assertContainsManifestWithContent(arm64OutputDirectory, "native_targets=linux_arm64") + assertContainsManifestWithContent(commonOutputDirectory, "native_targets=linux_x64 linux_arm64") + } +} diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetIdentityStringTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetIdentityStringTest.kt new file mode 100644 index 00000000000..7e06c4fd640 --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetIdentityStringTest.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.KonanTarget.* +import kotlin.test.Test +import kotlin.test.assertEquals + +class CommonizerTargetIdentityStringTest { + + @Test + fun leafTargets() { + KonanTarget.predefinedTargets.values.forEach { konanTarget -> + assertEquals(konanTarget.name, CommonizerTarget(konanTarget).identityString) + assertEquals(CommonizerTarget(konanTarget), parseCommonizerTarget(CommonizerTarget(konanTarget).identityString)) + } + } + + @Test + fun `simple shared targets are invariant under konanTarget order`() { + val macosFirst = CommonizerTarget(MACOS_X64, LINUX_X64) + val linuxFirst = CommonizerTarget(LINUX_X64, MACOS_X64) + + assertEquals(macosFirst, linuxFirst) + assertEquals(macosFirst.identityString, linuxFirst.identityString) + assertEquals(linuxFirst, parseCommonizerTarget(linuxFirst.identityString)) + assertEquals(macosFirst, parseCommonizerTarget(macosFirst.identityString)) + assertEquals(macosFirst, parseCommonizerTarget(linuxFirst.identityString)) + assertEquals(linuxFirst, parseCommonizerTarget(macosFirst.identityString)) + } + + @Test + fun `hierarchical commonizer targets`() { + val hierarchy = SharedCommonizerTarget( + CommonizerTarget(LINUX_X64, MACOS_X64), + CommonizerTarget(IOS_ARM64, IOS_X64) + ) + assertEquals(setOf(LINUX_X64, MACOS_X64, IOS_ARM64, IOS_X64), hierarchy.konanTargets) + assertEquals(hierarchy, parseCommonizerTarget(hierarchy.identityString)) + } + + @Test + fun `multilevel hierarchical commonizer targets`() { + val hierarchy = SharedCommonizerTarget( + SharedCommonizerTarget( + SharedCommonizerTarget( + SharedCommonizerTarget( + CommonizerTarget(LINUX_X64, MACOS_X64), + CommonizerTarget(IOS_X64, IOS_ARM64) + ), + CommonizerTarget(LINUX_ARM32_HFP) + ), + CommonizerTarget(LINUX_MIPSEL32) + ), + CommonizerTarget(WATCHOS_X86, WATCHOS_ARM64) + ) + + assertEquals(hierarchy, parseCommonizerTarget(hierarchy.identityString)) + } + + @Test + fun `parsing CommonizerTarget`() { + val target = parseCommonizerTarget("(x, (x, y, (a, b), (b, c)))") + assertEquals( + SharedCommonizerTarget( + LeafCommonizerTarget("x"), + SharedCommonizerTarget( + LeafCommonizerTarget("x"), + LeafCommonizerTarget("y"), + SharedCommonizerTarget( + LeafCommonizerTarget("a"), + LeafCommonizerTarget("b"), + ), + SharedCommonizerTarget( + LeafCommonizerTarget("b"), + LeafCommonizerTarget("c") + ) + ) + ), + target + ) + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 1`() { + parseCommonizerTarget("xxx,") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 2`() { + parseCommonizerTarget("") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 3`() { + parseCommonizerTarget("()") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 4`() { + parseCommonizerTarget("(xxx") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 5`() { + parseCommonizerTarget("xxx)") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 6`() { + parseCommonizerTarget("(xxx") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 7`() { + parseCommonizerTarget("(xxx yyy)") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 8`() { + parseCommonizerTarget(" ") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 9`() { + parseCommonizerTarget("xxx?") + } + + @Test(expected = IllegalArgumentException::class) + fun `fail parsing CommonizerTarget 10`() { + parseCommonizerTarget("(x, (x, y)") + } +} diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetPrettyNameTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetPrettyNameTest.kt new file mode 100644 index 00000000000..3891818b2e4 --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetPrettyNameTest.kt @@ -0,0 +1,88 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.junit.Test +import kotlin.test.assertEquals + +class CommonizerTargetPrettyNameTest { + + @Test + fun leafTargetNames() { + listOf( + Triple("foo", "[foo]", FOO), + Triple("bar", "[bar]", BAR), + Triple("baz_123", "[baz_123]", BAZ), + ).forEach { (name, prettyName, target: LeafCommonizerTarget) -> + assertEquals(name, target.name) + assertEquals(prettyName, target.prettyName) + } + } + + @Test + fun sharedTargetNames() { + listOf( + "[foo]" to SharedTarget(FOO), + "[bar, foo]" to SharedTarget(FOO, BAR), + "[bar, baz_123, foo]" to SharedTarget(FOO, BAR, BAZ), + "[bar, baz_123, foo, [bar, foo]]" to SharedTarget(FOO, BAR, BAZ, SharedTarget(FOO, BAR)) + ).forEach { (prettyName, target: SharedCommonizerTarget) -> + assertEquals(prettyName, target.prettyName) + } + } + + @Test + fun prettyCommonizedName() { + val sharedTarget = SharedTarget(FOO, BAR, BAZ) + listOf( + "[bar, baz_123, foo(*)]" to FOO, + "[bar(*), baz_123, foo]" to BAR, + "[bar, baz_123(*), foo]" to BAZ, + "[bar, baz_123, foo]" to sharedTarget, + ).forEach { (prettyCommonizerName, target: CommonizerTarget) -> + assertEquals(prettyCommonizerName, sharedTarget.prettyName(target)) + } + } + + @Test + fun prettyNestedName() { + val target = parseCommonizerTarget("(a, b, (c, (d, e)))") as SharedCommonizerTarget + + assertEquals( + "[a, b, [c, [d, e]]]", target.prettyName + ) + + assertEquals( + "[a, b, [c, [d, e(*)]]]", target.prettyName(LeafCommonizerTarget("e")) + ) + + assertEquals( + "[a, b, [c, [d, e](*)]]", target.prettyName(parseCommonizerTarget("(d, e)")) + ) + + assertEquals( + "[a, b, [c, [d, e]](*)]", target.prettyName(parseCommonizerTarget("(c, (d, e))")) + ) + + assertEquals( + "[a, b(*), [c, [d, e]]]", target.prettyName(LeafCommonizerTarget("b")) + ) + } + + @Test(expected = IllegalArgumentException::class) + fun sharedTargetNoInnerTargets() { + SharedCommonizerTarget(emptySet()) + } + + private companion object { + val FOO = LeafCommonizerTarget("foo") + val BAR = LeafCommonizerTarget("bar") + val BAZ = LeafCommonizerTarget("baz_123") + + @Suppress("TestFunctionName") + fun SharedTarget(vararg targets: CommonizerTarget) = SharedCommonizerTarget(linkedSetOf(*targets)) + } +} diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/utils/konanHome.kt b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/utils/konanHome.kt new file mode 100644 index 00000000000..941a49a4ccb --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/descriptors/commonizer/utils/konanHome.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.utils + +import java.io.File + +internal val konanHome: File + get() { + val konanHomePath = System.getenv("KONAN_HOME")?.toString() ?: error("Missing KONAN_HOME environment variable") + return File(konanHomePath) + } diff --git a/native/commonizer-api/testData/libcurl/linux_arm64/libcurl.klib b/native/commonizer-api/testData/libcurl/linux_arm64/libcurl.klib new file mode 100644 index 00000000000..a11828a1d12 Binary files /dev/null and b/native/commonizer-api/testData/libcurl/linux_arm64/libcurl.klib differ diff --git a/native/commonizer-api/testData/libcurl/linux_x64/libcurl.klib b/native/commonizer-api/testData/libcurl/linux_x64/libcurl.klib new file mode 100644 index 00000000000..2d09a208ebc Binary files /dev/null and b/native/commonizer-api/testData/libcurl/linux_x64/libcurl.klib differ diff --git a/native/commonizer/build.gradle.kts b/native/commonizer/build.gradle.kts index c9e124ef4b5..fcd6ca29fc2 100644 --- a/native/commonizer/build.gradle.kts +++ b/native/commonizer/build.gradle.kts @@ -16,11 +16,13 @@ configurations { dependencies { embedded(project(":kotlinx-metadata-klib")) { isTransitive = false } embedded(project(":kotlinx-metadata")) { isTransitive = false } + embedded(project(":native:kotlin-klib-commonizer-api")) { isTransitive = false } // N.B. The order of "kotlinx-metadata*" dependencies makes sense for runtime classpath // of the "runCommonizer" task. Please, don't mix them up. compileOnly(project(":kotlinx-metadata-klib")) { isTransitive = false } compileOnly(project(":kotlinx-metadata")) { isTransitive = false } + compileOnly(project(":native:kotlin-klib-commonizer-api")) { isTransitive = false } compileOnly(project(":compiler:cli-common")) compileOnly(project(":compiler:ir.serialization.common")) compileOnly(project(":compiler:frontend")) @@ -38,6 +40,7 @@ dependencies { testImplementation(projectTests(":compiler:tests-common")) testImplementation(project(":kotlinx-metadata-klib")) { isTransitive = false } testImplementation(project(":kotlinx-metadata")) { isTransitive = false } + testImplementation(project(":native:kotlin-klib-commonizer-api")) } val runCommonizer by tasks.registering(JavaExec::class) { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt index e19779bf7a4..f53a823693d 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt @@ -5,16 +5,19 @@ package org.jetbrains.kotlin.descriptors.commonizer +import org.jetbrains.kotlin.descriptors.commonizer.konan.TargetedNativeManifestDataProvider import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector class CommonizerParameters( + val resultsConsumer: ResultsConsumer, + val manifestDataProvider: TargetedNativeManifestDataProvider, val statsCollector: StatsCollector? = null, val progressLogger: ((String) -> Unit)? = null ) { // use linked hash map to preserve order - private val _targetProviders = LinkedHashMap() + private val _targetProviders = LinkedHashMap() val targetProviders: List get() = _targetProviders.values.toList() - val sharedTarget: SharedTarget get() = SharedTarget(_targetProviders.keys) + val sharedTarget: SharedCommonizerTarget get() = SharedCommonizerTarget(_targetProviders.keys) // common module dependencies (ex: Kotlin stdlib) var dependencyModulesProvider: ModulesProvider? = null @@ -30,14 +33,6 @@ class CommonizerParameters( return this } - private var _resultsConsumer: ResultsConsumer? = null - var resultsConsumer: ResultsConsumer - get() = _resultsConsumer ?: error("Results consumer has not been set") - set(value) { - check(_resultsConsumer == null) - _resultsConsumer = value - } - fun getCommonModuleNames(): Set { if (_targetProviders.size < 2) return emptySet() // too few targets diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt deleted file mode 100644 index b91271767a6..00000000000 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTarget.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.descriptors.commonizer - -import org.jetbrains.kotlin.konan.target.KonanTarget - -// N.B. TargetPlatform/SimplePlatform are non exhaustive enough to address both target platforms such as -// JVM, JS and concrete Kotlin/Native targets, e.g. macos_x64, ios_x64, linux_x64. -sealed class CommonizerTarget { - abstract val name: String - abstract val prettyName: String - - fun prettyCommonizedName(sharedTarget: SharedTarget): String = when { - this == sharedTarget -> prettyName - this in sharedTarget.targets -> sharedTarget.targets.joinToString(prefix = "[", postfix = "]") { - if (it == this) "${it.name}(*)" else it.name - } - else -> error("Target $prettyName is not in ${sharedTarget.prettyName}") - } -} - -data class LeafTarget(override val name: String, val konanTarget: KonanTarget? = null) : CommonizerTarget() { - override val prettyName get() = "[$name]" -} - -data class SharedTarget(val targets: Set) : CommonizerTarget() { - init { - require(targets.isNotEmpty()) - } - - override val name get() = targets.joinToString(prefix = "[", postfix = "]") { it.name } - override val prettyName get() = name -} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/KonanDistribution.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/KonanDistribution.kt new file mode 100644 index 00000000000..21ccb614359 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/KonanDistribution.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.konan.library.* +import java.io.File + +internal data class KonanDistribution(val root: File) + +internal val KonanDistribution.stdlib: File + get() = root.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) + +internal val KonanDistribution.klibDir: File + get() = root.resolve(KONAN_DISTRIBUTION_KLIB_DIR) + +internal val KonanDistribution.platformLibsDir: File + get() = klibDir.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/NativeLibraryLoader.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/NativeLibraryLoader.kt new file mode 100644 index 00000000000..78f88095602 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/NativeLibraryLoader.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion +import org.jetbrains.kotlin.backend.common.serialization.metadata.metadataVersion +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeLibrary +import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy +import org.jetbrains.kotlin.library.resolveSingleFileKlib +import org.jetbrains.kotlin.util.Logger +import java.io.File + +internal fun interface NativeLibraryLoader { + operator fun invoke(file: File): NativeLibrary +} + +internal class DefaultNativeLibraryLoader( + private val logger: Logger +) : NativeLibraryLoader { + override fun invoke(file: File): NativeLibrary { + val library = resolveSingleFileKlib( + libraryFile = org.jetbrains.kotlin.konan.file.File(file.path), + logger = logger, + strategy = ToolingSingleFileKlibResolveStrategy + ) + + if (library.versions.metadataVersion == null) + logger.fatal("Library does not have metadata version specified in manifest: $file") + + val metadataVersion = library.metadataVersion + if (metadataVersion?.isCompatible() != true) + logger.fatal( + """ + Library has incompatible metadata version ${metadataVersion ?: "\"unknown\""}: $file + Please make sure that all libraries passed to commonizer compatible metadata version ${KlibMetadataVersion.INSTANCE} + """.trimIndent() + ) + + return NativeLibrary(library) + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/ResultsConsumer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/ResultsConsumer.kt index cf9dfd5db2b..e3826298d8d 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/ResultsConsumer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/ResultsConsumer.kt @@ -5,9 +5,14 @@ package org.jetbrains.kotlin.descriptors.commonizer +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeSensitiveManifestData import org.jetbrains.kotlin.library.SerializedMetadata import java.io.File +internal fun buildResultsConsumer(init: ResultsConsumerBuilder.() -> Unit): ResultsConsumer { + return ResultsConsumerBuilder().apply(init).build() +} + interface ResultsConsumer { enum class Status { NOTHING_TO_DO, DONE } @@ -18,23 +23,70 @@ interface ResultsConsumer { override val libraryName: String get() = originalLocation.name } - class Commonized(override val libraryName: String, val metadata: SerializedMetadata) : ModuleResult() + class Commonized( + override val libraryName: String, val metadata: SerializedMetadata, val manifest: NativeSensitiveManifestData + ) : ModuleResult() } /** * Consume a single [ModuleResult] for the specified [CommonizerTarget]. */ - fun consume(target: CommonizerTarget, moduleResult: ModuleResult) + fun consume(target: CommonizerTarget, moduleResult: ModuleResult) = Unit /** * Mark the specified [CommonizerTarget] as fully consumed. * It's forbidden to make subsequent [consume] calls for fully consumed targets. */ - fun targetConsumed(target: CommonizerTarget) + fun targetConsumed(target: CommonizerTarget) = Unit /** * Notify that all results have been consumed. * It's forbidden to make any subsequent [consume] and [targetConsumed] calls after this call. */ - fun allConsumed(status: Status) + fun allConsumed(status: Status) = Unit +} + +internal class ResultsConsumerBuilder { + private var resultsConsumer: ResultsConsumer? = null + + infix fun add(consumer: ResultsConsumer) { + val resultsConsumer = this.resultsConsumer + if (resultsConsumer == null) { + this.resultsConsumer = consumer + return + } + this.resultsConsumer = resultsConsumer + consumer + } + + fun build(): ResultsConsumer { + return resultsConsumer ?: object : ResultsConsumer {} + } +} + +operator fun ResultsConsumer.plus(other: ResultsConsumer?): ResultsConsumer { + if (other == null) return this + if (this is CompositeResultsConsumer) { + return CompositeResultsConsumer(consumers + other) + } + return CompositeResultsConsumer(listOf(this, other)) +} + +private class CompositeResultsConsumer(val consumers: List) : ResultsConsumer { + override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) { + consumers.forEach { consumer -> + consumer.consume(target, moduleResult) + } + } + + override fun targetConsumed(target: CommonizerTarget) { + consumers.forEach { consumer -> + consumer.targetConsumed(target) + } + } + + override fun allConsumed(status: ResultsConsumer.Status) { + consumers.forEach { consumer -> + consumer.allConsumed(status) + } + } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt index 5f0b287a857..1a4343fd6d3 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt @@ -10,7 +10,7 @@ import org.jetbrains.kotlin.library.SerializedMetadata import java.io.File class TargetProvider( - val target: LeafTarget, + val target: LeafCommonizerTarget, val modulesProvider: ModulesProvider, val dependencyModulesProvider: ModulesProvider? ) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/DependencyLibrariesOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/DependencyLibrariesOptionType.kt new file mode 100644 index 00000000000..6b8dfbc284a --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/DependencyLibrariesOptionType.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.cli + +internal object DependencyLibrariesOptionType : LibrariesSetOptionType( + mandatory = true, + alias = "dependency-libraries", + description = "';' separated list of klib file paths that can be used as dependency" +) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/InputLibrariesOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/InputLibrariesOptionType.kt new file mode 100644 index 00000000000..1860655a6aa --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/InputLibrariesOptionType.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.cli + +internal object InputLibrariesOptionType : LibrariesSetOptionType( + mandatory = true, + alias = "input-libraries", + description = "';' separated list of klib file paths that will get commonized" +) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/LibrariesSetOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/LibrariesSetOptionType.kt new file mode 100644 index 00000000000..015cace78b5 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/LibrariesSetOptionType.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.cli + +import java.io.File + +internal abstract class LibrariesSetOptionType( + mandatory: Boolean, + alias: String, + description: String +) : OptionType>( + mandatory = mandatory, + alias = alias, + description = description +) { + override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option> { + if (rawValue.isBlank()) { + return Option(this, emptyList()) + } + return Option(this, rawValue.split(";").map(::File)) + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/OutputCommonizerTargetOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/OutputCommonizerTargetOptionType.kt new file mode 100644 index 00000000000..55f052137a0 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/OutputCommonizerTargetOptionType.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.cli + +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.parseCommonizerTarget + +internal object OutputCommonizerTargetOptionType : OptionType( + alias = "output-commonizer-target", + description = "Shared commonizer target representing the commonized output hierarchy", + mandatory = true +) { + override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option { + return try { + Option(this, parseCommonizerTarget(rawValue) as SharedCommonizerTarget) + } catch (t: Throwable) { + onError("Failed parsing output-commonizer-target ($rawValue): ${t.message}") + } + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/ProgressLogger.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/ProgressLogger.kt new file mode 100644 index 00000000000..90c130f6c39 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/ProgressLogger.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.cli + +import org.jetbrains.kotlin.util.Logger + +fun Logger.toProgressLogger(): Logger { + return ProgressLogger(this) +} + +private class ProgressLogger(private val logger: Logger) : Logger by logger { + override fun log(message: String) { + logger.log("* $message") + } + + override fun warning(message: String) { + logger.log("* $message") + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/StatsTypeOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/StatsTypeOptionType.kt index be731ff2d18..9cd3fb31477 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/StatsTypeOptionType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/StatsTypeOptionType.kt @@ -5,7 +5,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.cli -import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType +import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsType internal object StatsTypeOptionType : OptionType("log-stats", DESCRIPTION, mandatory = false) { override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/TaskType.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/TaskType.kt index c2030f04cb3..d9420f7818c 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/TaskType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/TaskType.kt @@ -40,7 +40,21 @@ internal enum class TaskType( NativeDistributionOptionType ), ::NativeDistributionListTargets - ); + ), + + NATIVE_KLIB_COMMONIZE( + "native-klib-commonize", + "Commonize any platform-specific libraries", + listOf( + NativeDistributionOptionType, + OutputOptionType, + InputLibrariesOptionType, + DependencyLibrariesOptionType, + OutputCommonizerTargetOptionType, + ), + ::NativeKlibCommonize + ) + ; companion object { fun getByAlias(alias: String) = values().firstOrNull { it.alias == alias } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt index e0ff95e4642..4af1aef8a9f 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt @@ -5,8 +5,20 @@ package org.jetbrains.kotlin.descriptors.commonizer.cli +import org.jetbrains.kotlin.descriptors.commonizer.* +import org.jetbrains.kotlin.descriptors.commonizer.konan.* +import org.jetbrains.kotlin.descriptors.commonizer.konan.CopyUnconsumedModulesAsIsConsumer +import org.jetbrains.kotlin.descriptors.commonizer.konan.CopyStdlibResultsConsumer import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer -import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType +import org.jetbrains.kotlin.descriptors.commonizer.konan.ModuleSerializer +import org.jetbrains.kotlin.descriptors.commonizer.repository.* +import org.jetbrains.kotlin.descriptors.commonizer.repository.EmptyRepository +import org.jetbrains.kotlin.descriptors.commonizer.repository.FilesRepository +import org.jetbrains.kotlin.descriptors.commonizer.repository.KonanDistributionRepository +import org.jetbrains.kotlin.descriptors.commonizer.repository.Repository +import org.jetbrains.kotlin.descriptors.commonizer.stats.FileStatsOutput +import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector +import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsType import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR import org.jetbrains.kotlin.konan.target.KonanTarget @@ -35,11 +47,52 @@ internal class NativeDistributionListTargets(options: Collection>) : T } } +internal class NativeKlibCommonize(options: Collection>) : Task(options) { + override val category: Category = Category.COMMONIZATION + + override fun execute(logPrefix: String) { + val distribution = KonanDistribution(getMandatory()) + val destination = getMandatory() + val targetLibraries = getMandatory, InputLibrariesOptionType>() + val dependencyLibraries = getMandatory, DependencyLibrariesOptionType>() + val outputCommonizerTarget = getMandatory() + val statsType = getOptional { it == "log-stats" } ?: StatsType.NONE + + val konanTargets = outputCommonizerTarget.konanTargets + val logger = CliLoggerAdapter(2) + val libraryLoader = DefaultNativeLibraryLoader(logger) + val statsCollector = StatsCollector(statsType, outputCommonizerTarget.konanTargets.toList()) + val repository = FilesRepository(targetLibraries.toSet(), libraryLoader) + + val resultsConsumer = buildResultsConsumer { + this add ModuleSerializer(destination, HierarchicalCommonizerOutputLayout) + this add CopyUnconsumedModulesAsIsConsumer( + repository, destination, konanTargets, NativeDistributionCommonizerOutputLayout, logger.toProgressLogger() + ) + this add LoggingResultsConsumer(outputCommonizerTarget, logger.toProgressLogger()) + } + + NativeDistributionCommonizer( + konanDistribution = distribution, + repository = repository, + dependencies = KonanDistributionRepository(distribution, outputCommonizerTarget.konanTargets, libraryLoader) + + FilesRepository(dependencyLibraries.toSet(), libraryLoader), + libraryLoader = libraryLoader, + targets = outputCommonizerTarget.konanTargets.toList(), + resultsConsumer = resultsConsumer, + statsCollector = statsCollector, + logger = logger + ).run() + + statsCollector?.writeTo(FileStatsOutput(destination, statsType.name.toLowerCase())) + } +} + internal class NativeDistributionCommonize(options: Collection>) : Task(options) { override val category get() = Category.COMMONIZATION override fun execute(logPrefix: String) { - val distribution = getMandatory() + val distribution = KonanDistribution(getMandatory()) val destination = getMandatory() val targets = getMandatory, NativeTargetsOptionType>() @@ -47,35 +100,43 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas val copyEndorsedLibs = getOptional { it == "copy-endorsed-libs" } ?: false val statsType = getOptional { it == "log-stats" } ?: StatsType.NONE - val targetNames = targets.joinToString { "[${it.name}]" } - val descriptionSuffix = estimateLibrariesCount(distribution, targets)?.let { " ($it items)" } ?: "" - val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix" + val logger = CliLoggerAdapter(2) + val libraryLoader = DefaultNativeLibraryLoader(logger) + val repository = KonanDistributionRepository(distribution, targets.toSet(), libraryLoader) + val statsCollector = StatsCollector(statsType, targets.toList()) + val resultsConsumer = buildResultsConsumer { + this add ModuleSerializer(destination, NativeDistributionCommonizerOutputLayout) + this add CopyUnconsumedModulesAsIsConsumer( + repository, destination, targets.toSet(), NativeDistributionCommonizerOutputLayout, logger.toProgressLogger() + ) + if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination, logger.toProgressLogger()) + if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination, logger.toProgressLogger()) + this add LoggingResultsConsumer(SharedCommonizerTarget(targets), logger.toProgressLogger()) + } + + val targetNames = targets.joinToString { "[${it.name}]" } + val descriptionSuffix = estimateLibrariesCount(repository, targets).let { " ($it items)" } + val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix" println(description) NativeDistributionCommonizer( - repository = distribution, + repository = repository, + konanDistribution = distribution, + dependencies = EmptyRepository, + libraryLoader = libraryLoader, targets = targets, - destination = destination, - copyStdlib = copyStdlib, - copyEndorsedLibs = copyEndorsedLibs, - statsType = statsType, - logger = CliLoggerAdapter(2) + resultsConsumer = resultsConsumer, + statsCollector = statsCollector, + logger = logger ).run() println("$description: Done") } companion object { - private fun estimateLibrariesCount(distribution: File, targets: List): Int? { - val targetNames = targets.map { it.name } - return distribution.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) - .listFiles() - ?.filter { it.name in targetNames } - ?.mapNotNull { it.listFiles() } - ?.flatMap { it.toList() } - ?.size + private fun estimateLibrariesCount(repository: Repository, targets: List): Int { + return targets.flatMap { repository.getLibraries(LeafCommonizerTarget(it)) }.count() } } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt index 6a684c9272a..a7bdcb64079 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt @@ -5,24 +5,24 @@ package org.jetbrains.kotlin.descriptors.commonizer.core -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget -import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.cir.CirRoot import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirRootFactory class RootCommonizer : AbstractStandardCommonizer() { - private val leafTargets = mutableSetOf() + private val leafTargets = mutableSetOf() override fun commonizationResult() = CirRootFactory.create( - target = SharedTarget(leafTargets) + target = SharedCommonizerTarget(leafTargets) ) override fun initialize(first: CirRoot) { - leafTargets += first.target as LeafTarget + leafTargets += first.target as LeafCommonizerTarget } override fun doCommonizeWith(next: CirRoot): Boolean { - leafTargets += next.target as LeafTarget + leafTargets += next.target as LeafCommonizerTarget return true } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt index fcb7eca39a3..e85620ef3c0 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt @@ -67,11 +67,11 @@ private fun serializeTarget(mergeResult: CirTreeMergeResult, targetIndex: Int, p val serializedMetadata = with(metadataModule.write(KLIB_FRAGMENT_WRITE_STRATEGY)) { SerializedMetadata(header, fragments, fragmentNames) } - - parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata)) + val manifestData = parameters.manifestDataProvider.getManifest(target, libraryName) + parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData)) } - if (target is LeafTarget) { + if (target is LeafCommonizerTarget) { mergeResult.missingModuleInfos.getValue(target).forEach { parameters.resultsConsumer.consume(target, ModuleResult.Missing(it.originalLocation)) } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyLibrariesFromKonanDistributionResultsConsumer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyLibrariesFromKonanDistributionResultsConsumer.kt new file mode 100644 index 00000000000..23f4002c501 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyLibrariesFromKonanDistributionResultsConsumer.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("FunctionName") + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.descriptors.commonizer.KonanDistribution +import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer +import org.jetbrains.kotlin.descriptors.commonizer.klibDir +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME +import org.jetbrains.kotlin.util.Logger +import java.io.File + +internal fun CopyStdlibResultsConsumer( + konanDistribution: KonanDistribution, + destination: File, + logger: Logger +): ResultsConsumer { + return CopyLibrariesFromKonanDistributionResultsConsumer( + konanDistribution, + destination, + invokeWhenCopied = { logger.log("Copied standard library") }, + copyFileIf = { file -> file.endsWith(KONAN_STDLIB_NAME) } + ) +} + +internal fun CopyEndorsedLibrairesResultsConsumer( + konanDistribution: KonanDistribution, + destination: File, + logger: Logger +): ResultsConsumer { + return CopyLibrariesFromKonanDistributionResultsConsumer( + konanDistribution, + destination, + invokeWhenCopied = { logger.log("Copied endorsed libraries") }, + copyFileIf = { file -> !file.endsWith(KONAN_STDLIB_NAME) } + ) +} + +private class CopyLibrariesFromKonanDistributionResultsConsumer( + private val konanDistribution: KonanDistribution, + private val destination: File, + private val invokeWhenCopied: () -> Unit = {}, + private val copyFileIf: (File) -> Boolean = { true } +) : ResultsConsumer { + override fun allConsumed(status: ResultsConsumer.Status) { + konanDistribution.klibDir + .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) + .listFiles().orEmpty() + .filter { it.isDirectory } + .filterNot(copyFileIf) + .forEach { libraryOrigin -> + val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name) + libraryOrigin.copyRecursively(libraryDestination) + } + invokeWhenCopied() + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyUnconsumedModulesAsIsConsumer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyUnconsumedModulesAsIsConsumer.kt new file mode 100644 index 00000000000..ce804095e4d --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/CopyUnconsumedModulesAsIsConsumer.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.descriptors.commonizer.* +import org.jetbrains.kotlin.descriptors.commonizer.repository.Repository +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.util.Logger +import java.io.File + +internal class CopyUnconsumedModulesAsIsConsumer( + private val repository: Repository, + private val destination: File, + private val targets: Set, + private val outputLayout: CommonizerOutputLayout, + private val logger: Logger? = null +) : ResultsConsumer { + + private val consumedTargets = mutableSetOf() + + override fun targetConsumed(target: CommonizerTarget) { + if (target is LeafCommonizerTarget) { + consumedTargets.add(target.konanTarget) + } + } + + override fun allConsumed(status: ResultsConsumer.Status) { + when (status) { + ResultsConsumer.Status.NOTHING_TO_DO -> targets.forEach(::copyTargetAsIs) + ResultsConsumer.Status.DONE -> targets.minus(consumedTargets).forEach(::copyTargetAsIs) + } + } + + private fun copyTargetAsIs(target: KonanTarget) { + val commonizerTarget = CommonizerTarget(target) + val libraries = repository.getLibraries(commonizerTarget) + val librariesDestination = outputLayout.getTargetDirectory(destination, commonizerTarget) + librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy + libraries.map { it.library.libraryFile.absolutePath }.map(::File).forEach { libraryFile -> + libraryFile.copyRecursively(destination.resolve(libraryFile.name)) + } + + logger?.log("Copied ${libraries.size} libraries for ${commonizerTarget.prettyName}") + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LoggingResultsConsumer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LoggingResultsConsumer.kt new file mode 100644 index 00000000000..b8c800740a9 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LoggingResultsConsumer.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.prettyName +import org.jetbrains.kotlin.util.Logger + +internal class LoggingResultsConsumer( + private val outputCommonizerTarget: SharedCommonizerTarget, private val logger: Logger +) : ResultsConsumer { + override fun targetConsumed(target: CommonizerTarget) { + logger.log("Written libraries for ${outputCommonizerTarget.prettyName(target)}") + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/ModuleSerializer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/ModuleSerializer.kt new file mode 100644 index 00000000000..19ec275bdc3 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/ModuleSerializer.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.descriptors.commonizer.* +import org.jetbrains.kotlin.library.SerializedMetadata +import org.jetbrains.kotlin.library.impl.BaseWriterImpl +import org.jetbrains.kotlin.library.impl.BuiltInsPlatform +import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter +import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl +import org.jetbrains.kotlin.util.Logger +import java.io.File + +internal class ModuleSerializer( + private val destination: File, + private val outputLayout: CommonizerOutputLayout, +) : ResultsConsumer { + override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) { + val librariesDestination = outputLayout.getTargetDirectory(destination, target) + when (moduleResult) { + is ResultsConsumer.ModuleResult.Commonized -> { + val libraryName = moduleResult.libraryName + val libraryDestination = librariesDestination.resolve(libraryName) + writeLibrary(moduleResult.metadata, moduleResult.manifest, libraryDestination) + } + is ResultsConsumer.ModuleResult.Missing -> { + val libraryName = moduleResult.libraryName + val missingModuleLocation = moduleResult.originalLocation + missingModuleLocation.copyRecursively(librariesDestination.resolve(libraryName)) + } + } + } +} + +private fun writeLibrary( + metadata: SerializedMetadata, + manifestData: NativeSensitiveManifestData, + libraryDestination: File +) { + val layout = org.jetbrains.kotlin.konan.file.File(libraryDestination.path).let { KotlinLibraryLayoutForWriter(it, it) } + val library = KotlinLibraryWriterImpl( + moduleName = manifestData.uniqueName, + versions = manifestData.versions, + builtInsPlatform = BuiltInsPlatform.NATIVE, + nativeTargets = emptyList(), // will be overwritten with NativeSensitiveManifestData.applyTo() below + nopack = true, + shortName = manifestData.shortName, + layout = layout + ) + library.addMetadata(metadata) + manifestData.applyTo(library.base as BaseWriterImpl) + library.commit() +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 76340070127..df23088222e 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -5,160 +5,91 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion -import org.jetbrains.kotlin.backend.common.serialization.metadata.metadataVersion import org.jetbrains.kotlin.descriptors.commonizer.* -import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType.* -import org.jetbrains.kotlin.descriptors.commonizer.stats.* +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.cli.toProgressLogger +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.* +import org.jetbrains.kotlin.descriptors.commonizer.repository.Repository +import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark import org.jetbrains.kotlin.konan.library.* import org.jetbrains.kotlin.konan.target.KonanTarget -import org.jetbrains.kotlin.library.KotlinLibrary -import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy -import org.jetbrains.kotlin.library.resolveSingleFileKlib import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.util.Logger -import java.io.File -import org.jetbrains.kotlin.konan.file.File as KFile -class NativeDistributionCommonizer( - private val repository: File, +internal class NativeDistributionCommonizer internal constructor( + private val konanDistribution: KonanDistribution, + private val repository: Repository, + private val dependencies: Repository, + private val libraryLoader: NativeLibraryLoader, private val targets: List, - private val destination: File, - private val copyStdlib: Boolean, - private val copyEndorsedLibs: Boolean, - private val statsType: StatsType, + private val resultsConsumer: ResultsConsumer, + private val statsCollector: StatsCollector?, private val logger: Logger ) { - enum class StatsType { RAW, AGGREGATED, NONE } private val clockMark = ResettableClockMark() fun run() { checkPreconditions() clockMark.reset() - - // 1. load libraries val allLibraries = loadLibraries() - - // 2. run commonization & write new libraries commonizeAndSaveResults(allLibraries) - logTotal() } - private fun checkPreconditions() { - if (!repository.isDirectory) - logger.fatal("Repository does not exist: $repository") - - when (targets.size) { - 0 -> logger.fatal("No targets specified") - 1 -> logger.fatal("Too few targets specified: $targets") - } - - when { - !destination.exists() -> destination.mkdirs() - !destination.isDirectory -> logger.fatal("Output already exists: $destination") - destination.walkTopDown().any { it != destination } -> logger.fatal("Output is not empty: $destination") - } - } - - private fun logProgress(message: String) = logger.log("* $message in ${clockMark.elapsedSinceLast()}") - - private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}") - private fun loadLibraries(): AllNativeLibraries { - val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) - val stdlib = NativeLibrary(loadLibrary(stdlibPath)) + val stdlib = libraryLoader(konanDistribution.stdlib) - val librariesByTargets = targets.associate { target -> - val leafTarget = LeafTarget(target.name, target) - - val platformLibs = leafTarget.platformLibrariesSource - .takeIf { it.isDirectory } - ?.listFiles() - ?.takeIf { it.isNotEmpty() } - ?.map { NativeLibrary(loadLibrary(it)) } - .orEmpty() - - if (platformLibs.isEmpty()) - logger.warning("No platform libraries found for target ${leafTarget.prettyName}. This target will be excluded from commonization.") - - leafTarget to NativeLibrariesToCommonize(platformLibs) + val librariesByTargets = targets.map(::LeafCommonizerTarget).associateWith { target -> + NativeLibrariesToCommonize(repository.getLibraries(target).toList()) } + librariesByTargets.forEach { (target, librariesToCommonize) -> + if (librariesToCommonize.libraries.isEmpty()) { + logger.warning("No platform libraries found for target $target. This target will be excluded from commonization.") + } + } logProgress("Read lazy (uninitialized) libraries") - return AllNativeLibraries(stdlib, librariesByTargets) } - private fun loadLibrary(location: File): KotlinLibrary { - if (!location.isDirectory) - logger.fatal("Library not found: $location") - - val library = resolveSingleFileKlib( - libraryFile = KFile(location.path), - logger = logger, - strategy = ToolingSingleFileKlibResolveStrategy - ) - - if (library.versions.metadataVersion == null) - logger.fatal("Library does not have metadata version specified in manifest: $location") - - val metadataVersion = library.metadataVersion - if (metadataVersion?.isCompatible() != true) - logger.fatal( - """ - Library has incompatible metadata version ${metadataVersion ?: "\"unknown\""}: $location - Please make sure that all libraries passed to commonizer compatible metadata version ${KlibMetadataVersion.INSTANCE} - """.trimIndent() - ) - - return library - } - private fun commonizeAndSaveResults(allLibraries: AllNativeLibraries) { - val statsCollector = when (statsType) { - RAW -> RawStatsCollector(targets) - AGGREGATED -> AggregatedStatsCollector(targets) - NONE -> null - } + val manifestProvider = TargetedNativeManifestDataProvider(allLibraries) - val parameters = CommonizerParameters(statsCollector, ::logProgress).apply { + val parameters = CommonizerParameters(resultsConsumer, manifestProvider, statsCollector, ::logProgress).apply { val storageManager = LockBasedStorageManager("Commonized modules") - - resultsConsumer = NativeDistributionResultsConsumer( - repository = repository, - originalLibraries = allLibraries, - destination = destination, - copyStdlib = copyStdlib, - copyEndorsedLibs = copyEndorsedLibs, - logProgress = ::logProgress - ) dependencyModulesProvider = NativeDistributionModulesProvider.forStandardLibrary(storageManager, allLibraries.stdlib) allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> if (librariesToCommonize.libraries.isEmpty()) return@forEach val modulesProvider = NativeDistributionModulesProvider.platformLibraries(storageManager, librariesToCommonize) + val dependencyModuleProvider = NativeDistributionModulesProvider.platformLibraries( + storageManager, NativeLibrariesToCommonize(dependencies.getLibraries(target).toList()), + ) addTarget( TargetProvider( target = target, modulesProvider = modulesProvider, - dependencyModulesProvider = null // stdlib is already set as common dependency + dependencyModulesProvider = dependencyModuleProvider ) ) } } runCommonization(parameters) - - statsCollector?.writeTo(FileStatsOutput(destination, statsType.name.toLowerCase())) } - private val LeafTarget.platformLibrariesSource: File - get() = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) - .resolve(name) + private fun checkPreconditions() { + when (targets.size) { + 0 -> logger.fatal("No targets specified") + 1 -> logger.fatal("Too few targets specified: $targets") + } + } + + private fun logProgress(message: String) = logger.toProgressLogger().log("$message in ${clockMark.elapsedSinceLast()}") + + private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}") } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt index 5dbd7c2546d..d977bb4c702 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt @@ -34,8 +34,6 @@ internal abstract class NativeDistributionModulesProvider(libraries: Collection< protected val moduleInfoMap: Map init { - check(libraries.isNotEmpty()) { "No libraries supplied" } - val libraryMap = mutableMapOf() val moduleInfoMap = mutableMapOf() diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionResultsConsumer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionResultsConsumer.kt deleted file mode 100644 index 4bed4d2c4e9..00000000000 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionResultsConsumer.kt +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.descriptors.commonizer.konan - -import com.intellij.util.containers.FactoryMap -import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer -import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult -import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status -import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget -import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR -import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME -import org.jetbrains.kotlin.library.SerializedMetadata -import org.jetbrains.kotlin.library.impl.BaseWriterImpl -import org.jetbrains.kotlin.library.impl.BuiltInsPlatform -import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter -import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl -import java.io.File -import org.jetbrains.kotlin.konan.file.File as KFile - -internal class NativeDistributionResultsConsumer( - private val repository: File, - private val originalLibraries: AllNativeLibraries, - private val destination: File, - private val copyStdlib: Boolean, - private val copyEndorsedLibs: Boolean, - private val logProgress: (String) -> Unit -) : ResultsConsumer { - private val allLeafTargets = originalLibraries.librariesByTargets.keys - - private val allLeafCommonizedTargets = originalLibraries.librariesByTargets.filterValues { it.libraries.isNotEmpty() }.keys - private val sharedTarget = SharedTarget(allLeafCommonizedTargets) - - private val consumedTargets = LinkedHashSet() - - private val cachedManifestProviders = FactoryMap.create { target -> - when (target) { - is LeafTarget -> originalLibraries.librariesByTargets.getValue(target) - is SharedTarget -> CommonNativeManifestDataProvider(originalLibraries.librariesByTargets.values) - } - } - - private val cachedLibrariesDestination = FactoryMap.create { target -> - val librariesDestination = when (target) { - is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(target.name) - is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) - } - - librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy - librariesDestination - } - - override fun consume(target: CommonizerTarget, moduleResult: ModuleResult) { - check(target in allLeafCommonizedTargets || target == sharedTarget) - consumedTargets += target - - serializeModule(target, moduleResult) - } - - override fun targetConsumed(target: CommonizerTarget) { - check(target in consumedTargets) - - logProgress("Written libraries for ${target.prettyCommonizedName(sharedTarget)}") - } - - override fun allConsumed(status: Status) { - // optimization: stdlib and endorsed libraries effectively remain the same across all Kotlin/Native targets, - // so they can be just copied to the new destination without running serializer - copyCommonStandardLibraries() - - when (status) { - Status.NOTHING_TO_DO -> { - // It may happen that all targets to be commonized (or at least all but one target) miss platform libraries. - // In such case commonizer will do nothing and raise a special status value 'NOTHING_TO_DO'. - // So, let's just copy platform libraries from the target where they are to the new destination. - allLeafTargets.forEach(::copyTargetAsIs) - } - Status.DONE -> { - // 'targetsToCopy' are some leaf targets with empty set of platform libraries - val targetsToCopy = allLeafTargets - consumedTargets.filterIsInstance() - targetsToCopy.forEach(::copyTargetAsIs) - } - } - } - - private fun copyCommonStandardLibraries() { - if (copyStdlib || copyEndorsedLibs) { - repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) - .listFiles() - ?.filter { it.isDirectory } - ?.let { - if (copyStdlib) { - if (copyEndorsedLibs) it else it.filter { dir -> dir.endsWith(KONAN_STDLIB_NAME) } - } else - it.filter { dir -> !dir.endsWith(KONAN_STDLIB_NAME) } - }?.forEach { libraryOrigin -> - val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name) - libraryOrigin.copyRecursively(libraryDestination) - } - - val what = listOfNotNull( - "standard library".takeIf { copyStdlib }, - "endorsed libraries".takeIf { copyEndorsedLibs } - ).joinToString(separator = " and ") - - logProgress("Copied $what") - } - } - - private fun copyTargetAsIs(leafTarget: LeafTarget) { - val librariesCount = originalLibraries.librariesByTargets.getValue(leafTarget).libraries.size - val librariesDestination = cachedLibrariesDestination.getValue(leafTarget) - - val librariesSource = leafTarget.platformLibrariesSource - if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination) - - logProgress("Copied $librariesCount libraries for ${leafTarget.prettyName}") - } - - private fun serializeModule(target: CommonizerTarget, moduleResult: ModuleResult) { - val librariesDestination = cachedLibrariesDestination.getValue(target) - - when (moduleResult) { - is ModuleResult.Commonized -> { - val libraryName = moduleResult.libraryName - - val manifestData = cachedManifestProviders.getValue(target).getManifest(libraryName) - val libraryDestination = librariesDestination.resolve(libraryName) - - writeLibrary(moduleResult.metadata, manifestData, libraryDestination) - } - is ModuleResult.Missing -> { - val libraryName = moduleResult.libraryName - val missingModuleLocation = moduleResult.originalLocation - - missingModuleLocation.copyRecursively(librariesDestination.resolve(libraryName)) - } - } - } - - private fun writeLibrary( - metadata: SerializedMetadata, - manifestData: NativeSensitiveManifestData, - libraryDestination: File - ) { - val layout = KFile(libraryDestination.path).let { KotlinLibraryLayoutForWriter(it, it) } - val library = KotlinLibraryWriterImpl( - moduleName = manifestData.uniqueName, - versions = manifestData.versions, - builtInsPlatform = BuiltInsPlatform.NATIVE, - nativeTargets = emptyList(), // will be overwritten with NativeSensitiveManifestData.applyTo() below - nopack = true, - shortName = manifestData.shortName, - layout = layout - ) - library.addMetadata(metadata) - manifestData.applyTo(library.base as BaseWriterImpl) - library.commit() - } - - private val LeafTarget.platformLibrariesSource: File - get() = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) - .resolve(name) -} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt index 557274101b2..d80d7ec9974 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt @@ -5,14 +5,33 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan +import com.intellij.util.containers.FactoryMap import gnu.trove.THashMap -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget +import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.library.KotlinLibrary +fun interface TargetedNativeManifestDataProvider { + fun getManifest(target: CommonizerTarget, libraryName: String): NativeSensitiveManifestData +} + internal interface NativeManifestDataProvider { fun getManifest(libraryName: String): NativeSensitiveManifestData } +internal fun TargetedNativeManifestDataProvider(libraries: AllNativeLibraries): TargetedNativeManifestDataProvider { + val cachedManifestProviders: Map = FactoryMap.create { target -> + when (target) { + is LeafCommonizerTarget -> libraries.librariesByTargets.getValue(target) + is SharedCommonizerTarget -> CommonNativeManifestDataProvider(libraries.librariesByTargets.values) + } + } + return TargetedNativeManifestDataProvider { target, libraryName -> + cachedManifestProviders.getValue(target).getManifest(libraryName) + } +} + /** * A separate Kotlin/Native library. */ @@ -37,7 +56,7 @@ internal class NativeLibrariesToCommonize(val libraries: List) : internal class AllNativeLibraries( val stdlib: NativeLibrary, - val librariesByTargets: Map + val librariesByTargets: Map ) internal class CommonNativeManifestDataProvider( diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeSensitiveManifestData.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeSensitiveManifestData.kt index 3d3ee40a5ef..a479c9c05b2 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeSensitiveManifestData.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeSensitiveManifestData.kt @@ -13,7 +13,7 @@ import org.jetbrains.kotlin.library.impl.BaseWriterImpl * The set of properties in manifest of Kotlin/Native library that should be * preserved in commonized libraries (both for "common" and platform-specific library parts). */ -internal data class NativeSensitiveManifestData( +data class NativeSensitiveManifestData( val uniqueName: String, val versions: KotlinLibraryVersioning, val dependencies: List, diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt index 27274c344c5..27cbc668f33 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt @@ -6,10 +6,8 @@ package org.jetbrains.kotlin.descriptors.commonizer.mergedtree import org.jetbrains.kotlin.descriptors.* -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget +import org.jetbrains.kotlin.descriptors.commonizer.* import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo -import org.jetbrains.kotlin.descriptors.commonizer.CommonizerParameters -import org.jetbrains.kotlin.descriptors.commonizer.TargetProvider import org.jetbrains.kotlin.descriptors.commonizer.cir.CirEntityId import org.jetbrains.kotlin.descriptors.commonizer.cir.CirName import org.jetbrains.kotlin.descriptors.commonizer.cir.CirPackageName @@ -49,7 +47,7 @@ class CirTreeMerger( ) { class CirTreeMergeResult( val root: CirRootNode, - val missingModuleInfos: Map> + val missingModuleInfos: Map> ) private val size = parameters.targetProviders.size diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/classifierContainers.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/classifierContainers.kt index 2fe350c6502..0f386bee1ea 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/classifierContainers.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/classifierContainers.kt @@ -8,7 +8,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.mergedtree import gnu.trove.THashMap import gnu.trove.THashSet import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.cir.CirEntityId import org.jetbrains.kotlin.descriptors.commonizer.cir.CirPackageName @@ -25,7 +25,7 @@ class CirKnownClassifiers( ) { // a shortcut for fast access val commonDependencies: CirProvidedClassifiers = - dependencies.filterKeys { it is SharedTarget }.values.singleOrNull() ?: CirProvidedClassifiers.EMPTY + dependencies.filterKeys { it is SharedCommonizerTarget }.values.singleOrNull() ?: CirProvidedClassifiers.EMPTY } interface CirCommonizedClassifiers { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/FilesRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/FilesRepository.kt new file mode 100644 index 00000000000..9719dddfdde --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/FilesRepository.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.repository + +import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.NativeLibraryLoader +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeLibrary +import org.jetbrains.kotlin.konan.target.KonanTarget +import java.io.File + +internal class FilesRepository( + private val libraryFiles: Set, + private val libraryLoader: NativeLibraryLoader +) : Repository { + + private val librariesByTarget: Map> by lazy { + libraryFiles + .map(libraryLoader::invoke) + .groupBy { library -> CommonizerTarget(library.manifestData.nativeTargets.map(::konanTargetOrThrow)) } + .mapValues { (_, list) -> list.toSet() } + } + + override fun getLibraries(target: LeafCommonizerTarget): Set { + return librariesByTarget[target].orEmpty() + } +} + +private fun konanTargetOrThrow(value: String): KonanTarget { + return KonanTarget.predefinedTargets[value] ?: error("Unexpected KonanTarget $value") +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt new file mode 100644 index 00000000000..3aa005bd904 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.repository + +import org.jetbrains.kotlin.descriptors.commonizer.KonanDistribution +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.NativeLibraryLoader +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeLibrary +import org.jetbrains.kotlin.descriptors.commonizer.platformLibsDir +import org.jetbrains.kotlin.konan.target.KonanTarget + +internal class KonanDistributionRepository( + konanDistribution: KonanDistribution, + targets: Set, + libraryLoader: NativeLibraryLoader, +) : Repository { + + private val librariesByTarget: Map>> = + targets.map(::LeafCommonizerTarget).associateWith { target -> + lazy { + konanDistribution.platformLibsDir + .resolve(target.name) + .takeIf { it.isDirectory } + ?.listFiles() + .orEmpty().toList() + .map { libraryLoader(it) } + .toSet() + } + } + + override fun getLibraries(target: LeafCommonizerTarget): Set { + return librariesByTarget[target]?.value ?: error("Missing target $target") + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/Repository.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/Repository.kt new file mode 100644 index 00000000000..5f7074b0a2b --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/Repository.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.repository + +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeLibrary + +internal interface Repository { + fun getLibraries(target: LeafCommonizerTarget): Set +} + +internal operator fun Repository.plus(other: Repository): Repository { + if (this is CompositeRepository) { + return CompositeRepository(this.repositories + other) + } + return CompositeRepository(listOf(this, other)) +} + +private class CompositeRepository(val repositories: Iterable) : Repository { + override fun getLibraries(target: LeafCommonizerTarget): Set { + return repositories.map { it.getLibraries(target) }.flatten().toSet() + } +} + +internal object EmptyRepository : Repository { + override fun getLibraries(target: LeafCommonizerTarget): Set { + return emptySet() + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/StatsCollector.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/StatsCollector.kt index 6a67558db9d..8adede045ed 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/StatsCollector.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/StatsCollector.kt @@ -5,6 +5,20 @@ package org.jetbrains.kotlin.descriptors.commonizer.stats +import org.jetbrains.kotlin.konan.target.KonanTarget + +fun StatsCollector(type: StatsType, targets: List): StatsCollector? { + return when (type) { + StatsType.RAW -> RawStatsCollector(targets) + StatsType.AGGREGATED -> AggregatedStatsCollector(targets) + StatsType.NONE -> null + } +} + +enum class StatsType { + RAW, AGGREGATED, NONE; +} + interface StatsCollector { data class StatsKey( val id: String, diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index 95af8cb91d4..fcad9a3e61f 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.SHARED_TARGET_NAME +import org.jetbrains.kotlin.descriptors.commonizer.konan.TargetedNativeManifestDataProvider import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ClassCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.FunctionCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.collectMembers @@ -75,7 +76,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { runCommonization(analyzedModules.toCommonizerParameters(results)) assertEquals(Status.DONE, results.status) - val sharedTarget: SharedTarget = analyzedModules.sharedTarget + val sharedTarget: SharedCommonizerTarget = analyzedModules.sharedTarget assertEquals(sharedTarget, results.sharedTarget) val sharedModuleAsExpected: SerializedMetadata = analyzedModules.commonizedModules.getValue(sharedTarget) @@ -84,7 +85,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { assertModulesAreEqual(sharedModuleAsExpected, sharedModuleByCommonizer, sharedTarget) - val leafTargets: Set = analyzedModules.leafTargets + val leafTargets: Set = analyzedModules.leafTargets assertEquals(leafTargets, results.leafTargets) for (leafTarget in leafTargets) { @@ -116,18 +117,18 @@ private data class SourceModuleRoot( } private class SourceModuleRoots( - val originalRoots: Map, + val originalRoots: Map, val commonizedRoots: Map, val dependencyRoots: Map ) { - val leafTargets: Set = originalRoots.keys - val sharedTarget: SharedTarget + val leafTargets: Set = originalRoots.keys + val sharedTarget: SharedCommonizerTarget init { check(leafTargets.size >= 2) check(leafTargets.none { it.name == SHARED_TARGET_NAME }) - val sharedTargets = commonizedRoots.keys.filterIsInstance() + val sharedTargets = commonizedRoots.keys.filterIsInstance() check(sharedTargets.size == 1) sharedTarget = sharedTargets.single() @@ -140,10 +141,10 @@ private class SourceModuleRoots( companion object { fun load(dataDir: File): SourceModuleRoots = try { - val originalRoots = listRoots(dataDir, ORIGINAL_ROOTS_DIR).mapKeys { LeafTarget(it.key) } + val originalRoots = listRoots(dataDir, ORIGINAL_ROOTS_DIR).mapKeys { LeafCommonizerTarget(it.key) } val leafTargets = originalRoots.keys - val sharedTarget = SharedTarget(leafTargets) + val sharedTarget = SharedCommonizerTarget(leafTargets) fun getTarget(targetName: String): CommonizerTarget = if (targetName == SHARED_TARGET_NAME) sharedTarget else leafTargets.first { it.name == targetName } @@ -185,41 +186,41 @@ private class AnalyzedModules( val commonizedModules: Map, val dependencyModules: Map> ) { - val leafTargets: Set - val sharedTarget: SharedTarget + val leafTargets: Set + val sharedTarget: SharedCommonizerTarget init { originalModules.keys.let { targets -> check(targets.isNotEmpty()) - leafTargets = targets.filterIsInstance().toSet() + leafTargets = targets.filterIsInstance().toSet() check(targets.size == leafTargets.size) } - sharedTarget = SharedTarget(leafTargets) + sharedTarget = SharedCommonizerTarget(leafTargets) val allTargets = leafTargets + sharedTarget check(commonizedModules.keys == allTargets) check(allTargets.containsAll(dependencyModules.keys)) } - fun toCommonizerParameters(resultsConsumer: ResultsConsumer) = - CommonizerParameters().also { parameters -> - parameters.resultsConsumer = resultsConsumer - parameters.dependencyModulesProvider = dependencyModules[sharedTarget]?.let(MockModulesProvider::create) + fun toCommonizerParameters( + resultsConsumer: ResultsConsumer, manifestDataProvider: TargetedNativeManifestDataProvider = MockNativeManifestDataProvider() + ) = CommonizerParameters(resultsConsumer, manifestDataProvider).also { parameters -> + parameters.dependencyModulesProvider = dependencyModules[sharedTarget]?.let(MockModulesProvider::create) - leafTargets.forEach { leafTarget -> - val originalModule = originalModules.getValue(leafTarget) + leafTargets.forEach { leafTarget -> + val originalModule = originalModules.getValue(leafTarget) - parameters.addTarget( - TargetProvider( - target = leafTarget, - modulesProvider = MockModulesProvider.create(originalModule), - dependencyModulesProvider = dependencyModules[leafTarget]?.let(MockModulesProvider::create) - ) + parameters.addTarget( + TargetProvider( + target = leafTarget, + modulesProvider = MockModulesProvider.create(originalModule), + dependencyModulesProvider = dependencyModules[leafTarget]?.let(MockModulesProvider::create) ) - } + ) } + } companion object { fun create( @@ -242,7 +243,7 @@ private class AnalyzedModules( } private fun createDependencyModules( - sharedTarget: SharedTarget, + sharedTarget: SharedCommonizerTarget, dependencyRoots: Map, parentDisposable: Disposable ): Pair>, AnalyzedModuleDependencies> { @@ -263,7 +264,7 @@ private class AnalyzedModules( } private fun createModules( - sharedTarget: SharedTarget, + sharedTarget: SharedCommonizerTarget, moduleRoots: Map, dependencies: AnalyzedModuleDependencies, parentDisposable: Disposable, @@ -290,7 +291,7 @@ private class AnalyzedModules( } private fun createModule( - sharedTarget: SharedTarget, + sharedTarget: SharedCommonizerTarget, currentTarget: CommonizerTarget, moduleRoot: SourceModuleRoot, dependencies: AnalyzedModuleDependencies, @@ -338,7 +339,7 @@ private class AnalyzedModules( } private class DependenciesContainerImpl( - sharedTarget: SharedTarget, + sharedTarget: SharedCommonizerTarget, currentTarget: CommonizerTarget, dependencies: AnalyzedModuleDependencies ) : CommonDependenciesContainer { diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt index 0f4af9f10f1..106cd004c61 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt @@ -7,8 +7,11 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status +import org.jetbrains.kotlin.descriptors.commonizer.konan.CommonNativeManifestDataProvider +import org.jetbrains.kotlin.descriptors.commonizer.konan.TargetedNativeManifestDataProvider import org.jetbrains.kotlin.descriptors.commonizer.utils.MockResultsConsumer import org.jetbrains.kotlin.descriptors.commonizer.utils.MockModulesProvider +import org.jetbrains.kotlin.descriptors.commonizer.utils.MockNativeManifestDataProvider import org.junit.Test import kotlin.contracts.ExperimentalContracts import kotlin.test.assertEquals @@ -62,20 +65,19 @@ class CommonizerFacadeTest { ) companion object { - private fun Map>.toCommonizerParameters(resultsConsumer: ResultsConsumer) = - CommonizerParameters().also { parameters -> - parameters.resultsConsumer = resultsConsumer - - forEach { (targetName, moduleNames) -> - parameters.addTarget( - TargetProvider( - target = LeafTarget(targetName), - modulesProvider = MockModulesProvider.create(moduleNames), - dependencyModulesProvider = null - ) + private fun Map>.toCommonizerParameters( + resultsConsumer: ResultsConsumer, manifestDataProvider: TargetedNativeManifestDataProvider = MockNativeManifestDataProvider() + ) = CommonizerParameters(resultsConsumer, manifestDataProvider).also { parameters -> + forEach { (targetName, moduleNames) -> + parameters.addTarget( + TargetProvider( + target = LeafCommonizerTarget(targetName), + modulesProvider = MockModulesProvider.create(moduleNames), + dependencyModulesProvider = null ) - } + ) } + } private fun doTestNothingToCommonize(originalModules: Map>) { val results = MockResultsConsumer() diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetTest.kt deleted file mode 100644 index f56e791652f..00000000000 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerTargetTest.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.descriptors.commonizer - -import org.jetbrains.kotlin.konan.target.KonanTarget -import org.junit.Test -import kotlin.test.assertEquals - -class CommonizerTargetTest { - - @Test - fun leafTargetNames() { - listOf( - Triple("foo", "[foo]", FOO), - Triple("bar", "[bar]", BAR), - Triple("baz_123", "[baz_123]", BAZ), - ).forEach { (name, prettyName, target: LeafTarget) -> - assertEquals(name, target.name) - assertEquals(prettyName, target.prettyName) - } - } - - @Test - fun sharedTargetNames() { - listOf( - "[foo]" to SharedTarget(FOO), - "[foo, bar]" to SharedTarget(FOO, BAR), - "[foo, bar, baz_123]" to SharedTarget(FOO, BAR, BAZ), - "[foo, bar, baz_123, [foo, bar]]" to SharedTarget(FOO, BAR, BAZ, SharedTarget(FOO, BAR)) - ).forEach { (prettyName, target: SharedTarget) -> - assertEquals(prettyName, target.prettyName) - assertEquals(prettyName, target.name) - } - } - - @Test - fun prettyCommonizedName() { - val sharedTarget = SharedTarget(FOO, BAR, BAZ) - listOf( - "[foo(*), bar, baz_123]" to FOO, - "[foo, bar(*), baz_123]" to BAR, - "[foo, bar, baz_123(*)]" to BAZ, - "[foo, bar, baz_123]" to sharedTarget - ).forEach { (prettyCommonizerName, target: CommonizerTarget) -> - assertEquals(prettyCommonizerName, target.prettyCommonizedName(sharedTarget)) - } - } - - @Test(expected = IllegalStateException::class) - fun prettyCommonizedNameFailure() { - FOO.prettyCommonizedName(SharedTarget(BAR, BAZ)) - } - - @Test(expected = IllegalArgumentException::class) - fun sharedTargetNoInnerTargets() { - SharedTarget(emptySet()) - } - - private companion object { - val FOO = LeafTarget("foo") - val BAR = LeafTarget("bar", KonanTarget.IOS_X64) - val BAZ = LeafTarget("baz_123", KonanTarget.MACOS_X64) - - @Suppress("TestFunctionName") - fun SharedTarget(vararg targets: CommonizerTarget) = SharedTarget(linkedSetOf(*targets)) - } -} diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt index a78996b49fb..c0ec2144ae1 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt @@ -6,8 +6,8 @@ package org.jetbrains.kotlin.descriptors.commonizer.core import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget -import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.cir.CirRoot import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirRootFactory import org.jetbrains.kotlin.konan.target.KonanTarget @@ -17,114 +17,86 @@ class RootCommonizerTest : AbstractCommonizerTest() { @Test fun allAreNative() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("ios_x64", KonanTarget.IOS_X64), - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64), - LeafTarget("ios_arm32", KonanTarget.IOS_ARM32) + LeafCommonizerTarget(KonanTarget.IOS_X64), + LeafCommonizerTarget(KonanTarget.IOS_ARM64), + LeafCommonizerTarget(KonanTarget.IOS_ARM32) ) ).toMock(), - LeafTarget("ios_x64", KonanTarget.IOS_X64).toMock(), - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64).toMock(), - LeafTarget("ios_arm32", KonanTarget.IOS_ARM32).toMock() + LeafCommonizerTarget(KonanTarget.IOS_X64).toMock(), + LeafCommonizerTarget(KonanTarget.IOS_ARM64).toMock(), + LeafCommonizerTarget(KonanTarget.IOS_ARM32).toMock() ) @Test fun jvmAndNative1() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("jvm1"), - LeafTarget("ios_x64", KonanTarget.IOS_X64), - LeafTarget("jvm2") + LeafCommonizerTarget("jvm1"), + LeafCommonizerTarget(KonanTarget.IOS_X64), + LeafCommonizerTarget("jvm2") ) ).toMock(), - LeafTarget("jvm1").toMock(), - LeafTarget("ios_x64", KonanTarget.IOS_X64).toMock(), - LeafTarget("jvm2").toMock() + LeafCommonizerTarget("jvm1").toMock(), + LeafCommonizerTarget(KonanTarget.IOS_X64).toMock(), + LeafCommonizerTarget("jvm2").toMock() ) @Test fun jvmAndNative2() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("ios_x64", KonanTarget.IOS_X64), - LeafTarget("jvm"), - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64) + LeafCommonizerTarget(KonanTarget.IOS_X64), + LeafCommonizerTarget("jvm"), + LeafCommonizerTarget(KonanTarget.IOS_ARM64) ) ).toMock(), - LeafTarget("ios_x64", KonanTarget.IOS_X64).toMock(), - LeafTarget("jvm").toMock(), - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64).toMock() + LeafCommonizerTarget(KonanTarget.IOS_X64).toMock(), + LeafCommonizerTarget("jvm").toMock(), + LeafCommonizerTarget(KonanTarget.IOS_ARM64).toMock() ) @Test fun noNative1() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("default1"), - LeafTarget("default2"), - LeafTarget("default3") + LeafCommonizerTarget("default1"), + LeafCommonizerTarget("default2"), + LeafCommonizerTarget("default3") ) ).toMock(), - LeafTarget("default1").toMock(), - LeafTarget("default2").toMock(), - LeafTarget("default3").toMock() + LeafCommonizerTarget("default1").toMock(), + LeafCommonizerTarget("default2").toMock(), + LeafCommonizerTarget("default3").toMock() ) @Test fun noNative2() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("jvm1"), - LeafTarget("default"), - LeafTarget("jvm2") + LeafCommonizerTarget("jvm1"), + LeafCommonizerTarget("default"), + LeafCommonizerTarget("jvm2") ) ).toMock(), - LeafTarget("jvm1").toMock(), - LeafTarget("default").toMock(), - LeafTarget("jvm2").toMock() + LeafCommonizerTarget("jvm1").toMock(), + LeafCommonizerTarget("default").toMock(), + LeafCommonizerTarget("jvm2").toMock() ) @Test fun noNative3() = doTestSuccess( - expected = SharedTarget( + expected = SharedCommonizerTarget( setOf( - LeafTarget("jvm1"), - LeafTarget("jvm2"), - LeafTarget("jvm3") + LeafCommonizerTarget("jvm1"), + LeafCommonizerTarget("jvm2"), + LeafCommonizerTarget("jvm3") ) ).toMock(), - LeafTarget("jvm1").toMock(), - LeafTarget("jvm2").toMock(), - LeafTarget("jvm3").toMock() - ) - - @Test(expected = ObjectsNotEqual::class) - fun misconfiguration1() = doTestSuccess( - expected = SharedTarget( - setOf( - LeafTarget("ios_x64", KonanTarget.IOS_X64), - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64), - LeafTarget("ios_arm32", KonanTarget.IOS_ARM32) - ) - ).toMock(), - LeafTarget("ios_x64").toMock(), // KonanTarget is missing here! - LeafTarget("ios_arm64", KonanTarget.IOS_ARM64).toMock(), - LeafTarget("ios_arm32", KonanTarget.IOS_ARM32).toMock() - ) - - @Test(expected = ObjectsNotEqual::class) - fun misconfiguration2() = doTestSuccess( - expected = SharedTarget( - setOf( - LeafTarget("jvm1"), - LeafTarget("jvm2"), - LeafTarget("jvm3") - ) - ).toMock(), - LeafTarget("jvm1", KonanTarget.IOS_X64).toMock(), // mistakenly specified KonanTarget! - LeafTarget("jvm2").toMock(), - LeafTarget("jvm3").toMock() + LeafCommonizerTarget("jvm1").toMock(), + LeafCommonizerTarget("jvm2").toMock(), + LeafCommonizerTarget("jvm3").toMock() ) override fun createCommonizer() = RootCommonizer() diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/TypeCommonizerTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/TypeCommonizerTest.kt index 7da9705f162..61a5a32e0be 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/TypeCommonizerTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/TypeCommonizerTest.kt @@ -7,8 +7,8 @@ package org.jetbrains.kotlin.descriptors.commonizer.core import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.TypeAliasDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget -import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget +import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.cir.CirEntityId import org.jetbrains.kotlin.descriptors.commonizer.cir.CirType import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory @@ -537,7 +537,7 @@ class TypeCommonizerTest : AbstractCommonizerTest() { fun areEqual(classifiers: CirKnownClassifiers, a: CirType, b: CirType): Boolean = TypeCommonizer(classifiers).run { commonizeWith(a) && commonizeWith(b) } - private val FAKE_SHARED_TARGET = SharedTarget(setOf(LeafTarget("a"), LeafTarget("b"))) + private val FAKE_SHARED_TARGET = SharedCommonizerTarget(setOf(LeafCommonizerTarget("a"), LeafCommonizerTarget("b"))) private fun CirKnownClassifiers.classNode(classId: CirEntityId, computation: () -> CirClassNode) = commonized.classNode(classId) ?: computation() diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt index 3d2eefde44e..72709bef6c6 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils import kotlinx.metadata.klib.KlibModuleMetadata import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget +import org.jetbrains.kotlin.descriptors.commonizer.identityString import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator.Mismatch import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator.Result @@ -38,7 +39,7 @@ fun assertModulesAreEqual(reference: SerializedMetadata, generated: SerializedMe val digitCount = mismatches.size.toString().length val failureMessage = buildString { - appendLine("${mismatches.size} mismatches found while comparing reference module ${referenceModule.name} (A) and generated module ${generatedModule.name} (B) for target ${target.prettyName}:") + appendLine("${mismatches.size} mismatches found while comparing reference module ${referenceModule.name} (A) and generated module ${generatedModule.name} (B) for target ${target.identityString}:") mismatches.forEachIndexed { index, mismatch -> appendLine((index + 1).toString().padStart(digitCount, ' ') + ". " + mismatch) } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt index 03a7f70c316..9169e2b0fe6 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt @@ -16,9 +16,12 @@ import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.cir.CirEntityId import org.jetbrains.kotlin.descriptors.commonizer.cir.CirName import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory +import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeSensitiveManifestData +import org.jetbrains.kotlin.descriptors.commonizer.konan.TargetedNativeManifestDataProvider import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.* import org.jetbrains.kotlin.descriptors.impl.AbstractTypeAliasDescriptor import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl +import org.jetbrains.kotlin.library.KotlinLibraryVersioning import org.jetbrains.kotlin.library.SerializedMetadata import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.parentOrNull @@ -204,8 +207,8 @@ internal class MockResultsConsumer : ResultsConsumer { val modulesByTargets: Map> get() = _modulesByTargets.mapValues { it.value.values } - val sharedTarget: SharedTarget by lazy { modulesByTargets.keys.filterIsInstance().single() } - val leafTargets: Set by lazy { modulesByTargets.keys.filterIsInstance().toSet() } + val sharedTarget: SharedCommonizerTarget by lazy { modulesByTargets.keys.filterIsInstance().single() } + val leafTargets: Set by lazy { modulesByTargets.keys.filterIsInstance().toSet() } private val finishedTargets = mutableSetOf() @@ -232,3 +235,25 @@ internal class MockResultsConsumer : ResultsConsumer { this.status = status } } + +fun MockNativeManifestDataProvider( + uniqueName: String = "mock", + versions: KotlinLibraryVersioning = KotlinLibraryVersioning(null, null, null, null, null), + dependencies: List = emptyList(), + isInterop: Boolean = true, + packageFqName: String? = "mock", + exportForwardDeclarations: List = emptyList(), + nativeTargets: Collection = emptyList(), + shortName: String? = "mock" +): TargetedNativeManifestDataProvider = TargetedNativeManifestDataProvider { target, libraryName -> + NativeSensitiveManifestData( + uniqueName = uniqueName, + versions = versions, + dependencies = dependencies, + isInterop = isInterop, + packageFqName = packageFqName, + exportForwardDeclarations = exportForwardDeclarations, + nativeTargets = nativeTargets, + shortName = shortName + ) +} diff --git a/settings.gradle b/settings.gradle index 0d56a3fe779..0cf60573b00 100644 --- a/settings.gradle +++ b/settings.gradle @@ -136,6 +136,7 @@ include ":benchmarks", ":native:kotlin-native-utils", ":native:frontend.native", ":native:kotlin-klib-commonizer", + ":native:kotlin-klib-commonizer-api", ":native:kotlin-klib-commonizer-embeddable", ":jps-plugin", ":kotlin-jps-plugin", @@ -492,6 +493,7 @@ project(':kotlin-util-klib-metadata').projectDir = "$rootDir/compiler/util-klib- project(':native:kotlin-native-utils').projectDir = "$rootDir/native/utils" as File project(':native:frontend.native').projectDir = "$rootDir/native/frontend" as File project(':native:kotlin-klib-commonizer').projectDir = "$rootDir/native/commonizer" as File +project(":native:kotlin-klib-commonizer-api").projectDir = "$rootDir/native/commonizer-api" as File project(':native:kotlin-klib-commonizer-embeddable').projectDir = "$rootDir/native/commonizer-embeddable" as File project(':kotlin-jps-plugin').projectDir = "$rootDir/prepare/jps-plugin" as File project(':idea:idea-android-output-parser').projectDir = "$rootDir/idea/idea-android/idea-android-output-parser" as File @@ -569,7 +571,7 @@ project(':plugins:jvm-abi-gen').projectDir = "$rootDir/plugins/jvm-abi-gen" as F project(':plugins:jvm-abi-gen-embeddable').projectDir = "$rootDir/plugins/jvm-abi-gen/embeddable" as File project(":dukat").projectDir = "$rootDir/libraries/tools/dukat" as File -project(':js:js.tests').projectDir = "$rootDir/js/js.tests" as File +project(':js:js.tests').projectDir = "$rootDir/js/js.tests" as File project(':js:js.engines').projectDir = "$rootDir/js/js.engines" as File project(':kotlinx-serialization-compiler-plugin').projectDir = file("$rootDir/plugins/kotlin-serialization/kotlin-serialization-compiler")