diff --git a/build.gradle.kts b/build.gradle.kts index a5b22f42de3..b35559872f8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -180,7 +180,7 @@ extra["versions.r8"] = "1.5.70" extra["versions.ktor-network"] = "1.0.1" if (!project.hasProperty("versions.kotlin-native")) { - extra["versions.kotlin-native"] = "1.4-dev-14579" + extra["versions.kotlin-native"] = "1.4-M2-dev-14852" } val intellijUltimateEnabled by extra(project.kotlinBuildProperties.intellijUltimateEnabled) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NativePlatformLibsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NativePlatformLibsIT.kt new file mode 100644 index 00000000000..c4b37fa1ba6 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NativePlatformLibsIT.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle + +import org.jetbrains.kotlin.gradle.util.modify +import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.presetName +import org.jetbrains.kotlin.konan.util.DependencyDirectories +import org.junit.Assume +import org.junit.Test + +class NativePlatformLibsIT : BaseGradleIT() { + + private val oldCompilerVersion = "1.3.61" + private val currentCompilerVersion = NativeCompilerDownloader.DEFAULT_KONAN_VERSION + + private fun platformLibrariesProject(vararg targets: String): Project = + transformProjectWithPluginsDsl("native-platform-libraries").apply { + setupWorkingDir() + gradleBuildScript().appendText( + targets.joinToString(prefix = "\n", separator = "\n") { + "kotlin.$it()" + } + ) + } + + private fun deleteInstalledCompilers() { + // Clean existing installation directories. + val osName = HostManager.simpleOsName() + val oldCompilerDir = DependencyDirectories.localKonanDir.resolve("kotlin-native-$osName-$oldCompilerVersion") + val currentCompilerDir = DependencyDirectories.localKonanDir.resolve("kotlin-native-$osName-$currentCompilerVersion") + + for (compilerDirectory in listOf(oldCompilerDir, currentCompilerDir)) { + compilerDirectory.deleteRecursively() + } + } + + @Test + fun testNoGenerationForOldCompiler() = with(platformLibrariesProject("linuxX64")) { + deleteInstalledCompilers() + + // Check that we don't run the library generator for old compiler distributions where libraries are prebuilt. + // Don't run the build to reduce execution time. + build("tasks", "-Pkotlin.native.version=1.3.60") { + assertSuccessful() + assertNotContains("Generate platform libraries for ") + } + } + + @Test + fun testLibrariesGeneration() { + deleteInstalledCompilers() + + val rootProject = Project("native-platform-libraries").apply { + embedProject(Project("native-platform-libraries"), renameTo = "subproject") + gradleBuildScript().modify(::transformBuildScriptWithPluginsDsl) + + gradleBuildScript().appendText("\nkotlin.linuxX64()\n") + gradleBuildScript("subproject").appendText("\nkotlin.linuxArm64()\n") + } + + with(rootProject) { + // Check that platform libraries are correctly generated for both root project and a subproject. + build("assemble") { + assertSuccessful() + assertContains("Generate platform libraries for linux_x64") + assertContains("Generate platform libraries for linux_arm64") + } + + // Check that we don't generate libraries during a second run. Don't clean to reduce execution time. + build("assemble") { + assertSuccessful() + assertNotContains("Generate platform libraries for ") + } + } + } + + @Test + fun testNoGenerationForUnsupportedHost() { + deleteInstalledCompilers() + + val unsupportedTarget = when { + HostManager.hostIsMac -> KonanTarget.MINGW_X64 + else -> KonanTarget.IOS_X64 + } + + platformLibrariesProject(unsupportedTarget.presetName).build("assemble") { + assertSuccessful() + assertNotContains("Generate platform libraries for ") + } + } + + @Test + fun testRerunGeneratorIfCacheKindChanged() { + // Currently we can generate caches only for macos_x64 and ios_x64. + Assume.assumeTrue(HostManager.hostIsMac) + + deleteInstalledCompilers() + + with(platformLibrariesProject("iosX64")) { + // Build Mac libraries without caches. + build("tasks") { + assertSuccessful() + assertContains("Generate platform libraries for ios_x64") + } + + // Change cache kind and check that platform libraries generator was executed. + build("tasks", "-Pkotlin.native.cacheKind=static") { + assertSuccessful() + assertContains("Precompile platform libraries for ios_x64 (precompilation: static)") + } + } + } + + @Test + fun testCanUsePrebuiltDistribution() = with(platformLibrariesProject("linuxX64")) { + deleteInstalledCompilers() + + build("assemble", "-Pkotlin.native.distribution.type=prebuilt") { + assertSuccessful() + assertContainsRegex("Kotlin/Native distribution: .*kotlin-native-prebuilt-(macos|linux|windows)".toRegex()) + assertNotContains("Generate platform libraries for ") + } + } + + @Test + fun testSettingGenerationMode() = with(platformLibrariesProject("linuxX64")) { + deleteInstalledCompilers() + + // Check that user can change generation mode used by the cinterop tool. + build("tasks", "-Pkotlin.native.platform.libraries.mode=metadata") { + assertSuccessful() + assertContainsRegex("Run tool: \"generatePlatformLibraries\" with args: .* -mode metadata".toRegex()) + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt index 5dfd3741bc8..f146a6d3a60 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt @@ -1805,20 +1805,6 @@ class NewMultiplatformIT : BaseGradleIT() { assertFalse(output.contains("Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.")) } - build("tasks", "-Pkotlin.native.restrictedDistribution=true") { - assertSuccessful() - val dists = output.lineSequence().filter { - it.contains("Kotlin/Native distribution: ") - } - - // Restricted distribution is available for Mac hosts only. - if (HostManager.hostIsMac) { - assertTrue(dists.all { it.contains("-restricted-") }) - } else { - assertTrue(dists.none { it.contains("-restricted-") }) - } - } - // This directory actually doesn't contain a K/N distribution // but we still can run a project configuration and check logs. val currentDir = projectDir.absolutePath diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/build.gradle.kts new file mode 100644 index 00000000000..09b67c3bc28 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/build.gradle.kts @@ -0,0 +1,22 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget + +plugins { + kotlin("multiplatform").version("") +} + +repositories { + mavenLocal() + jcenter() +} + +kotlin { + val nativeMain by sourceSets.creating { + dependsOn(sourceSets["commonMain"]) + } + + targets.withType(KotlinNativeTarget::class.java).all { + compilations["main"].defaultSourceSet.dependsOn(nativeMain) + } + + // Targets are added by test methods. +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/gradle.properties b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/gradle.properties new file mode 100644 index 00000000000..e9cc339258a --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/gradle.properties @@ -0,0 +1,2 @@ +kotlin.native.disableCompilerDaemon=true +kotlin.native.cacheKind=none \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/settings.gradle.kts b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/settings.gradle.kts new file mode 100644 index 00000000000..9f205d955cb --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/settings.gradle.kts @@ -0,0 +1,7 @@ +pluginManagement { + repositories { + mavenLocal() + jcenter() + gradlePluginPortal() + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/src/nativeMain/kotlin/main.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/src/nativeMain/kotlin/main.kt new file mode 100644 index 00000000000..043d37542ad --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/native-platform-libraries/src/nativeMain/kotlin/main.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +import platform.posix.* + +fun main() { + printf("Hello world %d", 42) +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/nativeToolRunners.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/nativeToolRunners.kt index ac397d08060..511ab7eeb06 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/nativeToolRunners.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/nativeToolRunners.kt @@ -137,3 +137,14 @@ internal class KotlinNativeCompilerRunner(project: Project) : KotlinNativeToolRu internal class KotlinNativeKlibRunner(project: Project) : KotlinNativeToolRunner("klib", project) { override val mustRunViaExec get() = project.disableKonanDaemon } + +/** Platform libraries generation tool. */ +internal class KotlinNativeLibraryGenerationRunner(project: Project) : + KotlinNativeToolRunner("generatePlatformLibraries", project) +{ + // Library generator starts the interop tool which cannot be executed in daemon. + override val mustRunViaExec: Boolean get() = true + + // The library generator works for a long time so enabling C2 can improve performance. + override val disableC2: Boolean = false +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt index 75227aebd3f..58c08671678 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt @@ -15,11 +15,14 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.* import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinAndroidTarget import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsSingleTargetPreset import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaTarget +import org.jetbrains.kotlin.gradle.plugin.mpp.isAtLeast import org.jetbrains.kotlin.gradle.targets.js.calculateJsCompilerType import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsTargetDsl import org.jetbrains.kotlin.gradle.targets.js.ir.KotlinJsIrSingleTargetPreset import org.jetbrains.kotlin.gradle.utils.lowerCamelCaseName +import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.konan.target.KonanTarget import kotlin.reflect.KClass private const val KOTLIN_PROJECT_EXTENSION_NAME = "kotlin" @@ -231,4 +234,25 @@ enum class ExplicitApiMode(private val cliOption: String) { Disabled("disabled"); fun toCompilerArg() = "-Xexplicit-api=$cliOption" +} + +enum class NativeDistributionType(val suffix: String?) { + REGULAR(null) { + override fun isAvailableFor(host: KonanTarget, version: CompilerVersion) = true + }, + RESTRICTED("restricted") { + override fun isAvailableFor(host: KonanTarget, version: CompilerVersion): Boolean = + host == KonanTarget.MACOS_X64 && version.major == 1 && version.minor == 3 + }, + PREBUILT("prebuilt") { + override fun isAvailableFor(host: KonanTarget, version: CompilerVersion): Boolean = + version.isAtLeast(1, 4, 0) + }; + + abstract fun isAvailableFor(host: KonanTarget, version: CompilerVersion): Boolean + + companion object { + fun byCompilerArgument(argument: String): NativeDistributionType? = + values().firstOrNull { it.name.equals(argument, ignoreCase = true) } + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index 295cfa249a3..d8393f915c4 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -19,9 +19,11 @@ package org.jetbrains.kotlin.gradle.plugin import org.gradle.api.Project import org.jetbrains.kotlin.gradle.dsl.Coroutines import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind +import org.jetbrains.kotlin.gradle.dsl.NativeDistributionType import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.Companion.jsCompilerProperty import org.jetbrains.kotlin.gradle.targets.native.DisabledNativeTargetsReporter import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile +import org.jetbrains.kotlin.gradle.tasks.CacheBuilder import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.utils.SingleWarningPerBuild @@ -119,13 +121,35 @@ internal class PropertiesProvider private constructor(private val project: Proje get() = booleanProperty("kotlin.tests.individualTaskReports") /** - * Forces using a "restricted" distribution of Kotlin/Native. - * - * A restricted distribution is available for MacOS only and doesn't contain platform libraries. - * If a host platform is not MacOS, the flag is ignored. + * Allow a user to choose distribution type. The following distribution types are available: + * - regular - The default distribution. Includes all platform libraries in 1.3 and generates them at the user side in 1.4. + * - restricted - Doesn't include Apple platform libraries. Available for MacOS only and in 1.3 only. + * - prebuilt - Includes all platform libraries. Available in 1.4 only. Used to workaround possible problems with library generation at the use side. */ - val nativeRestrictedDistribution: Boolean? - get() = booleanProperty("kotlin.native.restrictedDistribution") + val nativeDistributionType: NativeDistributionType? + get() { + var result = property("kotlin.native.distribution.type")?.let { + NativeDistributionType.byCompilerArgument(it) + } + + val deprecatedRestricted = booleanProperty("kotlin.native.restrictedDistribution") + if (result == null && deprecatedRestricted != null) { + SingleWarningPerBuild.show( + project, + "Project property 'kotlin.native.restrictedDistribution' is deprecated. Please use 'kotlin.native.distribution.type=restricted' instead" + ) + result = if (deprecatedRestricted) NativeDistributionType.RESTRICTED else NativeDistributionType.REGULAR + } + + return result + } + + /** + * Allows a user to force a particular cinterop mode for platform libraries generation. Available modes: sourcecode, metadata. + * A main purpose of this property is working around potential problems with the metadata mode. + */ + val nativePlatformLibrariesMode: String? + get() = property("kotlin.native.platform.libraries.mode") /** * Allows a user to provide a local Kotlin/Native distribution instead of a downloaded one. @@ -155,7 +179,7 @@ internal class PropertiesProvider private constructor(private val project: Proje * Dependencies caching strategy. The default is static. */ val nativeCacheKind: NativeCacheKind - get() = property("kotlin.native.cacheKind")?.let { NativeCacheKind.byCompilerArgument(it) } ?: NativeCacheKind.STATIC + get() = property("kotlin.native.cacheKind")?.let { NativeCacheKind.byCompilerArgument(it) } ?: CacheBuilder.DEFAULT_CACHE_KIND /** * Generate kotlin/js external declarations from all .d.ts files found in npm modules diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt index ce2385561d0..81262fc1037 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/KotlinNativeTargetPreset.kt @@ -3,9 +3,8 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -@file:Suppress("PackageDirectoryMismatch") - // Old package for compatibility +@file:Suppress("PackageDirectoryMismatch") package org.jetbrains.kotlin.gradle.plugin.mpp @@ -13,14 +12,20 @@ import org.gradle.BuildAdapter import org.gradle.api.Project import org.gradle.api.invocation.Gradle import org.jetbrains.kotlin.compilerRunner.konanHome +import org.jetbrains.kotlin.compilerRunner.konanVersion +import org.jetbrains.kotlin.gradle.dsl.NativeDistributionType +import org.jetbrains.kotlin.gradle.dsl.NativeDistributionType.REGULAR import org.jetbrains.kotlin.gradle.plugin.* import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.gradle.targets.native.DisabledNativeTargetsReporter +import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator import org.jetbrains.kotlin.gradle.targets.native.internal.setUpKotlinNativePlatformDependencies import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject +import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.target.HostManager import org.jetbrains.kotlin.konan.target.KonanTarget +import java.io.File abstract class AbstractKotlinNativeTargetPreset( private val name: String, @@ -51,6 +56,13 @@ abstract class AbstractKotlinNativeTargetPreset( } else { logger.info("User-provided Kotlin/Native distribution: $konanHome") } + + val distributionType = PropertiesProvider(project).nativeDistributionType + if (konanVersion.isAtLeast(1, 4, 0) && + (distributionType == null || distributionType == REGULAR) + ) { + PlatformLibrariesGenerator(project, konanTarget).generatePlatformLibsIfNeeded() + } } protected abstract fun createTargetConfigurator(): KotlinTargetConfigurator @@ -134,3 +146,9 @@ internal val KonanTarget.enabledOnCurrentHost internal val AbstractKotlinNativeCompilation.isMainCompilation: Boolean get() = name == KotlinCompilation.MAIN_COMPILATION_NAME + +// KonanVersion doesn't provide an API to compare versions, +// so we have to transform it to KotlinVersion first. +// Note: this check doesn't take into account the meta version (release, eap, dev). +internal fun CompilerVersion.isAtLeast(major: Int, minor: Int, patch: Int): Boolean = + KotlinVersion(this.major, this.minor, this.maintenance).isAtLeast(major, minor, patch) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/NativeCompilerDownloader.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/NativeCompilerDownloader.kt index 5cf31a3d7db..d05329672ec 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/NativeCompilerDownloader.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/NativeCompilerDownloader.kt @@ -12,6 +12,7 @@ import org.gradle.api.file.FileTree import org.gradle.api.logging.Logger import org.jetbrains.kotlin.compilerRunner.KotlinNativeCompilerRunner import org.jetbrains.kotlin.compilerRunner.konanVersion +import org.jetbrains.kotlin.gradle.dsl.NativeDistributionType import org.jetbrains.kotlin.gradle.logging.kotlinInfo import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.konan.CompilerVersion @@ -26,7 +27,7 @@ class NativeCompilerDownloader( ) { companion object { - internal val DEFAULT_KONAN_VERSION: CompilerVersion by lazy { + val DEFAULT_KONAN_VERSION: CompilerVersion by lazy { CompilerVersion.fromString(loadPropertyFromResources("project.properties", "kotlin.native.version")) } @@ -39,18 +40,22 @@ class NativeCompilerDownloader( private val logger: Logger get() = project.logger - // We provide restricted distributions only for Mac. - private val restrictedDistribution: Boolean - get() = HostManager.hostIsMac && PropertiesProvider(project).nativeRestrictedDistribution ?: false + private val distributionType: NativeDistributionType + get() = PropertiesProvider(project).nativeDistributionType?.takeIf { + it.isAvailableFor(HostManager.host, compilerVersion) + } ?: NativeDistributionType.REGULAR private val simpleOsName: String get() = HostManager.simpleOsName() private val dependencyName: String - get() = if (restrictedDistribution) { - "kotlin-native-restricted-$simpleOsName" - } else { - "kotlin-native-$simpleOsName" + get() { + val dependencySuffix = distributionType.suffix + return if (dependencySuffix != null) { + "kotlin-native-$dependencySuffix-$simpleOsName" + } else { + "kotlin-native-$simpleOsName" + } } private val dependencyNameWithVersion: String diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/PlatformLibrariesGenerator.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/PlatformLibrariesGenerator.kt new file mode 100644 index 00000000000..4177e5f8b5b --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/PlatformLibrariesGenerator.kt @@ -0,0 +1,220 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.native.internal + +import org.gradle.api.Project +import org.jetbrains.kotlin.compilerRunner.KotlinNativeLibraryGenerationRunner +import org.jetbrains.kotlin.compilerRunner.konanCacheKind +import org.jetbrains.kotlin.compilerRunner.konanHome +import org.jetbrains.kotlin.gradle.dsl.NativeCacheKind +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider +import org.jetbrains.kotlin.gradle.tasks.CacheBuilder +import org.jetbrains.kotlin.gradle.tasks.addArg +import org.jetbrains.kotlin.gradle.utils.lifecycleWithDuration +import org.jetbrains.kotlin.konan.library.KONAN_PLATFORM_LIBS_NAME_PREFIX +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.customerDistribution +import org.jetbrains.kotlin.konan.util.visibleName +import java.io.File +import java.util.* +import java.util.concurrent.ConcurrentHashMap + +internal class PlatformLibrariesGenerator(val project: Project, val konanTarget: KonanTarget) { + + private val distribution = + customerDistribution(project.konanHome) + + private val platformLibsDirectory = + File(distribution.platformLibs(konanTarget)).absoluteFile + + private val defDirectory = + File(distribution.platformDefs(konanTarget.family)).absoluteFile + + private val shouldBuildCaches: Boolean = + CacheBuilder.cacheWorksFor(konanTarget) && project.konanCacheKind != NativeCacheKind.NONE + + private val mode: String? by lazy { + PropertiesProvider(project).nativePlatformLibrariesMode + } + + private val presentDefs: Set by lazy { + defDirectory + .listFiles { file -> file.extension == "def" }.orEmpty() + .map { it.nameWithoutExtension }.toSet() + } + + private fun Set.toPlatformLibNames(): Set = + mapTo(mutableSetOf()) { "$KONAN_PLATFORM_LIBS_NAME_PREFIX$it" } + + /** + * Checks that all platform libs for [konanTarget] actually exist in the [distribution]. + */ + private fun checkLibrariesInDistribution(): Boolean { + val presentPlatformLibs = platformLibsDirectory + .listFiles { file -> file.isDirectory }.orEmpty() + .map { it.name }.toSet() + + // TODO: Check that all directories in presentPlatformLibs are real klibs when klib componentization is merged. + return presentDefs.toPlatformLibNames().all { it in presentPlatformLibs } + } + + /** + * Check that caches for all platform libs for [konanTarget] actually exist in the cache directory. + */ + private fun checkCaches(): Boolean { + if (!shouldBuildCaches) { + return true + } + + val cacheDirectory = CacheBuilder.getRootCacheDirectory( + File(project.konanHome), konanTarget, true, project.konanCacheKind + ) + val presentCacheFiles = cacheDirectory.listFiles().orEmpty().map { it.name }.toSet() + return presentDefs.toPlatformLibNames().all { + CacheBuilder.getCacheFileName(it, project.konanCacheKind, konanTarget) in presentCacheFiles + } + } + + /** + * We store directories where platform libraries were detected/generated earlier + * during this build to avoid redundant distribution checks. + */ + private val alreadyProcessed: PlatformLibsInfo + get() = project.rootProject.extensions.extraProperties.run { + if (!has(GENERATED_LIBS_PROPERTY_NAME)) { + set(GENERATED_LIBS_PROPERTY_NAME, PlatformLibsInfo()) + } + @Suppress("UNCHECKED_CAST") + get(GENERATED_LIBS_PROPERTY_NAME) as PlatformLibsInfo + } + + private fun runGenerationTool() = with(project) { + val args = mutableListOf("-target", konanTarget.visibleName) + if (logger.isInfoEnabled) { + args.add("-verbose") + } + + // We can generate caches using either [CacheBuilder] or the library generator. Using CacheBuilder allows + // keeping all the caching logic in one place while the library generator speeds up building caches because + // it works in parallel. We use the library generator for now due to performance reasons. + // + // TODO: Supporting Gradle Worker API (or other parallelization) in the CacheBuilder and enabling + // the compiler daemon for interop will allow switching to the CacheBuilder without performance penalty. + // Alternatively we can rely on the library generator tool in the CacheBuilder and eliminate a separate + // logic for library caching there. + if (shouldBuildCaches) { + args.addArg("-cache-kind", konanCacheKind.produce!!) + args.addArg( + "-cache-directory", + CacheBuilder.getRootCacheDirectory(File(konanHome), konanTarget, true, konanCacheKind).absolutePath + ) + args.addArg("-cache-arg", "-g") + } + + mode?.let { + args.addArg("-mode", it) + } + + KotlinNativeLibraryGenerationRunner(this).run(args) + } + + fun generatePlatformLibsIfNeeded(): Unit = with(project) { + if (!HostManager(distribution).isEnabled(konanTarget)) { + // We cannot generate libs on a machine that doesn't support the requested target. + return + } + + // Don't run the generator if libraries/caches for this target were already built during this Gradle invocation. + val alreadyGenerated = alreadyProcessed.isGenerated(platformLibsDirectory) + val alreadyCached = alreadyProcessed.isCached(platformLibsDirectory, project.konanCacheKind) + if ((alreadyGenerated && alreadyCached) || !defDirectory.exists()) { + return + } + + // Check if libraries/caches for this target already exist (requires reading from disc). + val platformLibsAreReady = checkLibrariesInDistribution() + if (platformLibsAreReady) { + alreadyProcessed.setGenerated(platformLibsDirectory) + } + + val cachesAreReady = checkCaches() + if (cachesAreReady) { + alreadyProcessed.setCached(platformLibsDirectory, project.konanCacheKind) + } + + val generationMessage = when { + !platformLibsAreReady && !cachesAreReady -> + "Generate and precompile platform libraries for $konanTarget (precompilation: ${project.konanCacheKind.visibleName})" + platformLibsAreReady && !cachesAreReady -> + "Precompile platform libraries for $konanTarget (precompilation: ${project.konanCacheKind.visibleName})" + !platformLibsAreReady && cachesAreReady -> + "Generate platform libraries for $konanTarget" + else -> { + // Both caches and libraries exist thus there is no need to run the generator. + return + } + } + + logger.lifecycle(generationMessage) + logger.lifecycleWithDuration("$generationMessage finished,") { + runGenerationTool() + } + + val librariesAreActuallyGenerated = checkLibrariesInDistribution() + assert(librariesAreActuallyGenerated) { "Some platform libraries were not generated" } + if (librariesAreActuallyGenerated) { + alreadyProcessed.setGenerated(platformLibsDirectory) + } + + val librariesAreActuallyCached = checkCaches() + assert(librariesAreActuallyCached) { "Some platform libraries were not precompiled" } + if (librariesAreActuallyCached) { + alreadyProcessed.setCached(platformLibsDirectory, project.konanCacheKind) + } + } + + private class PlatformLibsInfo { + private val generated: MutableSet = Collections.newSetFromMap(ConcurrentHashMap()) + private val cached: ConcurrentHashMap> = ConcurrentHashMap() + + private fun cached(cacheKind: NativeCacheKind): MutableSet = cached.getOrPut(cacheKind) { + Collections.newSetFromMap(ConcurrentHashMap()) + } + + /** + * Are platform libraries in the given directory (e.g. /klib/platform/ios_x64) generated. + */ + fun isGenerated(path: File): Boolean = + generated.contains(path) + + /** + * Register that platform libraries in the given directory are generated. + */ + fun setGenerated(path: File) { + generated.add(path) + } + + /** + * Are platform libraries in the given directory (e.g. /klib/platform/ios_x64) cached with the given cache kind. + */ + fun isCached(path: File, kind: NativeCacheKind): Boolean = + kind == NativeCacheKind.NONE || cached(kind).contains(path) + + /** + * Register that platform libraries in the give directory are cached with the given cache kind. + */ + fun setCached(path: File, kind: NativeCacheKind) { + if (kind != NativeCacheKind.NONE) { + cached(kind).add(path) + } + } + } + + companion object { + private const val GENERATED_LIBS_PROPERTY_NAME = "org.jetbrains.kotlin.native.platform.libs.info" + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt index f5587de1fc3..721b6de3d33 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt @@ -646,13 +646,12 @@ open class KotlinNativeLink : AbstractKotlinNativeCompile { val allDependencies = mutableSetOf() @@ -799,9 +797,7 @@ class CacheBuilder(val project: Project, val binary: NativeBinary) { } private val String.cachedName - get() = konanCacheKind.outputKind?.let { - "${it.prefix(compilation.konanTarget)}${this}-cache${it.suffix(compilation.konanTarget)}" - } ?: error("No output for kind $konanCacheKind") + get() = getCacheFileName(this, konanCacheKind, compilation.konanTarget) private fun ensureCompilerProvidedLibPrecached(platformLibName: String, platformLibs: Map, visitedLibs: MutableSet) { if (platformLibName in visitedLibs) @@ -834,11 +830,9 @@ class CacheBuilder(val project: Project, val binary: NativeBinary) { ensureCompilerProvidedLibPrecached(platformLibName, platformLibs, visitedLibs) } - private val KonanTarget.cacheWorks - get() = this == KonanTarget.IOS_X64 || this == KonanTarget.MACOS_X64 fun buildCompilerArgs(): List = mutableListOf().apply { - if (konanCacheKind != NativeCacheKind.NONE && !optimized && compilation.konanTarget.cacheWorks) { + if (konanCacheKind != NativeCacheKind.NONE && !optimized && cacheWorksFor(compilation.konanTarget)) { rootCacheDirectory.mkdirs() ensureCompilerProvidedLibsPrecached() add("-Xcache-directory=${rootCacheDirectory.absolutePath}") @@ -857,6 +851,24 @@ class CacheBuilder(val project: Project, val binary: NativeBinary) { add("-Xcache-directory=$cacheDirectory") } } + + companion object { + internal fun getRootCacheDirectory(konanHome: File, target: KonanTarget, debuggable: Boolean, cacheKind: NativeCacheKind): File { + require(cacheKind != NativeCacheKind.NONE) { "Usupported cache kind: ${NativeCacheKind.NONE}" } + val optionsAwareCacheName = "$target${if (debuggable) "-g" else ""}$cacheKind" + return konanHome.resolve("klib/cache/$optionsAwareCacheName") + } + + internal fun getCacheFileName(baseName: String, cacheKind: NativeCacheKind, konanTarget: KonanTarget): String = + cacheKind.outputKind?.let { + "${it.prefix(konanTarget)}${baseName}-cache${it.suffix(konanTarget)}" + } ?: error("No output for kind $cacheKind") + + internal fun cacheWorksFor(target: KonanTarget) = + target == KonanTarget.IOS_X64 || target == KonanTarget.MACOS_X64 + + internal val DEFAULT_CACHE_KIND: NativeCacheKind = NativeCacheKind.STATIC + } } open class CInteropProcess : DefaultTask() { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTest.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTest.kt index c661ad9792c..96e9ee5dd26 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTest.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTest.kt @@ -18,6 +18,7 @@ import org.jetbrains.kotlin.compilerRunner.konanVersion import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesClientSettings import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutionSpec import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutor.Companion.TC_PROJECT_PROPERTY +import org.jetbrains.kotlin.gradle.plugin.mpp.isAtLeast import org.jetbrains.kotlin.gradle.targets.native.internal.parseKotlinNativeStackTraceAsJvm import org.jetbrains.kotlin.gradle.tasks.KotlinTest import org.jetbrains.kotlin.konan.CompilerVersion @@ -102,12 +103,6 @@ abstract class KotlinNativeTest : KotlinTest() { @get:Internal protected abstract val testCommand: TestCommand - // KonanVersion doesn't provide an API to compare versions, - // so we have to transform it to KotlinVersion first. - // Note: this check doesn't take into account the meta version (release, eap, dev). - private fun CompilerVersion.isAtLeast(major: Int, minor: Int, patch: Int): Boolean = - KotlinVersion(this.major, this.minor, this.maintenance).isAtLeast(major, minor, patch) - override fun createTestExecutionSpec(): TCServiceMessagesTestExecutionSpec { val extendedForkOptions = DefaultProcessForkOptions(fileResolver) processOptions.copyTo(extendedForkOptions) diff --git a/native/utils/src/org/jetbrains/kotlin/konan/library/NativeLibraryConstants.kt b/native/utils/src/org/jetbrains/kotlin/konan/library/NativeLibraryConstants.kt index 84618661d65..ab175852eca 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/library/NativeLibraryConstants.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/library/NativeLibraryConstants.kt @@ -22,3 +22,6 @@ fun konanCommonLibraryPath(libraryName: String) = fun konanPlatformLibraryPath(libraryName: String, platform: String) = File(KONAN_DISTRIBUTION_KLIB_DIR, KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(platform).resolve(libraryName) + +// Used to provide unique names for platform libraries according to KT-36720. +const val KONAN_PLATFORM_LIBS_NAME_PREFIX = "org.jetbrains.kotlin.native.platform." diff --git a/native/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt b/native/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt index b990586c6da..40be6d46c12 100644 --- a/native/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt +++ b/native/utils/src/org/jetbrains/kotlin/konan/target/Distribution.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.konan.properties.Properties import org.jetbrains.kotlin.konan.properties.keepOnlyDefaultProfiles import org.jetbrains.kotlin.konan.properties.loadProperties import org.jetbrains.kotlin.konan.util.DependencyDirectories +import org.jetbrains.kotlin.konan.util.visibleName class Distribution( private val onlyDefaultProfiles: Boolean = false, @@ -90,6 +91,10 @@ class Distribution( fun runtime(target: KonanTarget) = runtimeFileOverride ?: "$stdlibDefaultComponent/targets/${target.visibleName}/native/runtime.bc" + fun platformDefs(family: Family) = "$konanHome/konan/platformDef/${family.visibleName}" + + fun platformLibs(target: KonanTarget) = "$klib/platform/${target.visibleName}" + val launcherFiles = listOf("launcher.bc") val dependenciesDir = DependencyDirectories.defaultDependenciesRoot.absolutePath